1
0
Fork 0
mirror of https://github.com/seejohnrun/haste-server.git synced 2024-12-25 03:26:59 +00:00

[SAT-1957] Convert haste-server to Typescript

This commit is contained in:
Yusuf Yilmaz 2022-05-27 11:00:09 +02:00
parent 68f6fe2b96
commit 9f45927593
38 changed files with 9825 additions and 1701 deletions

56
.eslintrc.js Normal file
View file

@ -0,0 +1,56 @@
module.exports = {
env: {
node: true,
},
extends: [
'eslint:recommended',
'plugin:@typescript-eslint/recommended',
'airbnb-base',
'airbnb-typescript/base',
'plugin:import/errors',
'plugin:import/warnings',
'plugin:import/typescript',
'prettier',
],
plugins: [
'import',
'@typescript-eslint'
],
settings: {
'import/parsers': {
'@typescript-eslint/parser': ['.ts'],
},
'import/resolver': {
node: {
extensions: ['.js', '.ts'],
moduleDirectory: ['node_modules', 'src/'],
},
typescript: {
alwaysTryTypes: true,
project: '.',
},
},
},
overrides: [
{
env: {
jest: true,
},
files: ['**/__tests__/**/*.[jt]s', '**/?(*.)+(spec|test).[jt]s'],
extends: ['plugin:jest/recommended'],
rules: {
'import/no-extraneous-dependencies': [
'off',
{ devDependencies: ['**/?(*.)+(spec|test).[jt]s'] },
],
camelcase: ['off'],
},
},
],
ignorePatterns: ['**/*.js', 'node_modules', 'dist'],
parserOptions: {
root: true,
tsconfigRootDir: __dirname,
project: ['./tsconfig.json'],
},
}

View file

@ -1,25 +0,0 @@
{
"env": {
"es6": true,
"node": true
},
"extends": "eslint:recommended",
"rules": {
"indent": [
"error",
2
],
"linebreak-style": [
"error",
"unix"
],
"quotes": [
"error",
"single"
],
"semi": [
"error",
"always"
]
}
}

1
.gitignore vendored
View file

@ -5,3 +5,4 @@ node_modules
data
*.DS_Store
docker-compose.override.yml
dist

8
.prettierrc.json Normal file
View file

@ -0,0 +1,8 @@
{
"tabWidth": 2,
"semi": false,
"trailingComma": "all",
"printWidth": 80,
"arrowParens": "avoid",
"singleQuote": true
}

View file

@ -1,16 +1,10 @@
{
"host": "0.0.0.0",
"port": 7777,
"keyLength": 10,
"maxLength": 400000,
"staticMaxAge": 86400,
"recompressStaticAssets": true,
"logging": [
{
"level": "verbose",
@ -18,11 +12,9 @@
"colorize": true
}
],
"keyGenerator": {
"type": "phonetic"
},
"rateLimits": {
"categories": {
"normal": {
@ -31,13 +23,10 @@
}
}
},
"storage": {
"type": "file"
},
"documents": {
"about": "./about.md"
}
}

1652
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -12,36 +12,66 @@
"email": "john.crepezzi@gmail.com",
"url": "http://seejohncode.com/"
},
"main": "haste",
"dependencies": {
"@google-cloud/datastore": "^6.6.2",
"aws-sdk": "^2.1142.0",
"busboy": "0.2.4",
"connect": "^3.7.0",
"connect-ratelimit": "0.0.7",
"connect-ratelimit": "^0.0.7",
"connect-route": "0.1.5",
"pg": "^8.0.0",
"dotenv": "^16.0.1",
"express": "^4.18.1",
"memcached": "^2.2.2",
"mongodb": "^4.6.0",
"pg": "^8.7.3",
"redis": "0.8.1",
"redis-url": "0.1.0",
"st": "^2.0.0",
"rethinkdbdash": "^2.3.31",
"st": "^3.0.0",
"uglify-js": "3.1.6",
"winston": "^2.0.0"
},
"devDependencies": {
"mocha": "^8.1.3"
"@types/busboy": "^1.5.0",
"@types/express": "^4.17.13",
"@types/memcached": "^2.2.7",
"@types/node": "^17.0.35",
"@types/pg": "^8.6.5",
"@types/uglify-js": "^3.13.2",
"@typescript-eslint/eslint-plugin": "^5.26.0",
"@typescript-eslint/parser": "^5.26.0",
"concurrently": "^7.2.1",
"eslint": "^8.10.0",
"eslint-config-airbnb": "^19.0.4",
"eslint-config-airbnb-typescript": "^17.0.0",
"eslint-config-prettier": "^8.5.0",
"eslint-import-resolver-typescript": "^2.7.1",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-jest": "^26.2.2",
"mocha": "^8.1.3",
"module-resolver": "^1.0.0",
"nodemon": "^2.0.16",
"npm-run-all": "^4.1.5",
"prettier": "^2.5.1",
"rimraf": "^3.0.2",
"ts-node": "^9.1.1",
"typescript": "^4.6.4"
},
"bundledDependencies": [],
"main": "haste",
"bin": {
"haste-server": "./server.js"
"haste-server": ".dist/src/server.js"
},
"files": [
"server.js",
"lib",
"src",
"static"
],
"directories": {
"lib": "./lib"
},
"scripts": {
"start": "node server.js",
"test": "mocha --recursive"
"test": "mocha --recursive",
"build": "rimraf dist && tsc --project ./",
"start:dev": "nodemon src/server.ts",
"start:prod": "node dist/src/server.js",
"lint": "eslint src --fix",
"types:check": "tsc --noEmit --pretty"
}
}

175
src/app.ts Normal file
View file

@ -0,0 +1,175 @@
import express, { Router, Express, Request } from 'express'
import * as fs from 'fs'
import * as winston from 'winston'
import uglify from 'uglify-js'
import connectSt from 'st'
import connectRateLimit from 'connect-ratelimit'
import getConfig from './lib/helpers/config'
import addLogging from './lib/helpers/log'
import build from './lib/document-handler/builder'
import DocumentHandler from './lib/document-handler'
import { Config } from './types/config'
import {
getStaticDirectory,
getStaticItemDirectory,
} from './lib/helpers/directory'
class App {
public server: Express
public config: Config
documentHandler?: DocumentHandler
constructor() {
this.config = getConfig()
this.server = express()
this.setLogging()
this.setDocumentHandler()
this.compressStaticAssets()
this.sendDocumentsToStore()
this.middlewares()
this.setRateLimits()
this.apiCalls()
this.staticPages()
}
middlewares() {
this.server.use(express.json())
}
setLogging() {
if (this.config.logging) {
addLogging(this.config)
}
}
setDocumentHandler = async () => {
this.documentHandler = await build(this.config)
}
apiCalls() {
const router = Router()
// get raw documents - support getting with extension
router.get('/raw/:id', async (request, response) =>
this.documentHandler?.handleRawGet(request, response),
)
router.head('/raw/:id', (request, response) =>
this.documentHandler?.handleRawGet(request, response),
)
// // add documents
router.post('/documents', (request, response) =>
this.documentHandler?.handlePost(request, response),
)
// get documents
router.get('/documents/:id', (request, response) =>
this.documentHandler?.handleGet(request, response),
)
router.head('/documents/:id', (request, response) =>
this.documentHandler?.handleGet(request, response),
)
this.server.use(router)
}
setRateLimits() {
if (this.config.rateLimits) {
this.config.rateLimits.end = true
this.server.use(connectRateLimit(this.config.rateLimits))
}
}
compressStaticAssets() {
// Compress the static javascript assets
if (this.config.recompressStaticAssets) {
const list = fs.readdirSync(getStaticDirectory(__dirname))
for (let j = 0; j < list.length; j += 1) {
const item = list[j]
if (
item.indexOf('.js') === item.length - 3 &&
item.indexOf('.min.js') === -1
) {
const dest = `${item.substring(
0,
item.length - 3,
)}.min${item.substring(item.length - 3)}`
const origCode = fs.readFileSync(
getStaticItemDirectory(__dirname, item),
'utf8',
)
fs.writeFileSync(
getStaticItemDirectory(__dirname, dest),
uglify.minify(origCode).code,
'utf8',
)
winston.info(`compressed ${item} into ${dest}`)
}
}
}
}
sendDocumentsToStore() {
// Send the static documents into the preferred store, skipping expirations
let documentPath
let data
Object.keys(this.config.documents).forEach(name => {
documentPath = this.config.documents[name]
data = fs.readFileSync(documentPath, 'utf8')
winston.info('loading static document', { name, path: documentPath })
if (data) {
this.documentHandler?.store.set(
name,
data,
cb => {
winston.debug('loaded static document', { success: cb })
},
true,
)
} else {
winston.warn('failed to load static document', {
name,
path: documentPath,
})
}
})
}
staticPages() {
// Otherwise, try to match static files
this.server.use(
connectSt({
path: getStaticDirectory(__dirname),
content: { maxAge: this.config.staticMaxAge },
passthrough: true,
index: false,
}),
)
// Then we can loop back - and everything else should be a token,
// so route it back to /
this.server.get('/:id', (request: Request, response, next) => {
request.sturl = '/'
next()
})
// And match index
this.server.use(
connectSt({
path: getStaticDirectory(__dirname),
content: { maxAge: this.config.staticMaxAge },
index: 'index.html',
}),
)
}
}
export default App

74
src/global.d.ts vendored Normal file
View file

@ -0,0 +1,74 @@
declare module 'rethinkdbdash' {
type Result = {
data: string
}
type Callback = (error: unknown, result?: Result) => void
interface RethinkRun {
run(callback: Callback)
}
type RethinkInsertObject = {
id: string
data: string
}
interface RethinkFunctions {
insert(object: RethinkInsertObject): RethinkRun
get(x: string): RethinkRun
}
export interface RethinkClient {
table(s: string): RethinkFunctions
}
// function rethink<T>(obj: T[]): RethinkArray<T>
function rethink<T>(obj: T): RethinkClient<T>
export = rethink
}
// export {}
// declare module 'connect-ratelimit' {
// export = connectRateLimit
// }
// declare namespace Express {
// export interface Request {
// sturl?: string
// }
// }
declare module 'connect-ratelimit' {
function connectRateLimit(
as: RateLimits,
): (
req: express.Request,
res: express.Response,
next: express.NextFunction,
) => void
export = connectRateLimit
}
declare namespace Express {
export interface Request {
sturl: string
}
}
declare module 'st' {
type ConnectSt = {
path: string
content: { maxAge : number }
passthrough? : boolean
index: boolean | string
}
function connectSt(st: ConnectSt): Middleware
export = connectSt
}

View file

@ -0,0 +1,21 @@
import type { Config } from '../../types/config'
import buildGenerator from '../key-generators/builder'
import buildStore from '../document-stores/builder'
import DocumentHandler from "./index"
const build = async (config: Config) => {
const storage = await buildStore(config)
const keyGenerator = await buildGenerator(config)
const documentHandler = new DocumentHandler({
store: storage,
config,
maxLength: config.maxLength,
keyLength: config.keyLength,
keyGenerator,
})
return documentHandler
}
export default build

View file

@ -0,0 +1,177 @@
import { Request, Response } from 'express'
import * as winston from 'winston'
import Busboy from 'busboy'
import type { Config } from '../../types/config'
import type { Store } from '../../types/store'
import type { KeyGenerator } from '../../types/key-generator'
import type { Document } from '../../types/document'
const defaultKeyLength = 10
class DocumentHandler {
keyLength: number
maxLength: number
public store: Store
keyGenerator: KeyGenerator
config: Config
constructor(options: Document) {
this.keyLength = options.keyLength || defaultKeyLength
this.maxLength = options.maxLength // none by default
this.store = options.store
this.config = options.config
this.keyGenerator = options.keyGenerator
}
public handleGet(request: Request, response: Response) {
const key = request.params.id.split('.')[0]
const skipExpire = !!this.config.documents[key]
this.store.get(
key,
ret => {
if (ret) {
winston.verbose('retrieved document', { key })
response.writeHead(200, { 'content-type': 'application/json' })
if (request.method === 'HEAD') {
response.end()
} else {
response.end(JSON.stringify({ data: ret, key }))
}
} else {
winston.warn('document not found', { key })
response.writeHead(404, { 'content-type': 'application/json' })
if (request.method === 'HEAD') {
response.end()
} else {
response.end(JSON.stringify({ message: 'Document not found.' }))
}
}
},
skipExpire,
)
}
public handlePost(request: Request, response: Response) {
// const this = this
let buffer = ''
let cancelled = false
// What to do when done
const onSuccess = () => {
// Check length
if (this.maxLength && buffer.length > this.maxLength) {
cancelled = true
winston.warn('document >maxLength', { maxLength: this.maxLength })
response.writeHead(400, { 'content-type': 'application/json' })
response.end(
JSON.stringify({ message: 'Document exceeds maximum length.' }),
)
return
}
// And then save if we should
this.chooseKey(key => {
this.store.set(key, buffer, res => {
if (res) {
winston.verbose('added document', { key })
response.writeHead(200, { 'content-type': 'application/json' })
response.end(JSON.stringify({ key }))
} else {
winston.verbose('error adding document')
response.writeHead(500, { 'content-type': 'application/json' })
response.end(JSON.stringify({ message: 'Error adding document.' }))
}
})
})
}
// If we should, parse a form to grab the data
const ct = request.headers['content-type']
if (ct && ct.split(';')[0] === 'multipart/form-data') {
const busboy = Busboy({ headers: request.headers })
busboy.on('field', (fieldname, val) => {
if (fieldname === 'data') {
buffer = val
}
})
busboy.on('finish', () => {
onSuccess()
})
request.pipe(busboy)
// Otherwise, use our own and just grab flat data from POST body
} else {
request.on('data', data => {
buffer += data.toString()
})
request.on('end', () => {
if (cancelled) {
return
}
onSuccess()
})
request.on('error', error => {
winston.error(`connection error: ${error.message}`)
response.writeHead(500, { 'content-type': 'application/json' })
response.end(JSON.stringify({ message: 'Connection error.' }))
cancelled = true
})
}
}
public handleRawGet(request: Request, response: Response) {
const key = request.params.id.split('.')[0]
const skipExpire = !!this.config.documents[key]
this.store.get(
key,
ret => {
if (ret) {
winston.verbose('retrieved raw document', { key })
response.writeHead(200, {
'content-type': 'text/plain; charset=UTF-8',
})
if (request.method === 'HEAD') {
response.end()
} else {
response.end(ret)
}
} else {
winston.warn('raw document not found', { key })
response.writeHead(404, { 'content-type': 'application/json' })
if (request.method === 'HEAD') {
response.end()
} else {
response.end(JSON.stringify({ message: 'Document not found.' }))
}
}
},
skipExpire,
)
}
chooseKey = (callback: { (key: string): void }) => {
const key = this.acceptableKey()
if (!key) return
this.store.get(
key,
(ret: string | boolean) => {
if (ret) {
this.chooseKey(callback)
} else {
callback(key)
}
},
true,
) // Don't bump expirations when key searching
}
acceptableKey = () => this.keyGenerator.createKey?.(this.keyLength)
}
export default DocumentHandler

View file

@ -0,0 +1,80 @@
import * as winston from 'winston'
import AWS from 'aws-sdk'
import type { Callback, Store } from '../../types/store'
import type { AmazonStoreConfig } from '../../types/config'
class AmazonS3DocumentStore implements Store {
bucket: string | undefined
client: AWS.S3
type: string
expire?: number | undefined
constructor(options: AmazonStoreConfig) {
this.expire = options.expire
this.bucket = options.bucket
this.type = options.type
this.client = new AWS.S3({ region: options.region })
}
get = (
key: string,
callback: Callback,
skipExpire?: boolean | undefined,
): void => {
if (!this.bucket) {
callback(false)
return
}
const req = {
Bucket: this.bucket,
Key: key,
}
this.client.getObject(req, (err, data) => {
if (err || !data.Body) {
callback(false)
} else {
callback(data.Body.toString('utf-8'))
if (this.expire && !skipExpire) {
winston.warn('amazon s3 store cannot set expirations on keys')
}
}
})
}
set = (
key: string,
data: string,
callback: Callback,
skipExpire?: boolean | undefined,
): void => {
if (!this.bucket) {
callback(false)
return
}
const req = {
Bucket: this.bucket,
Key: key,
Body: data as AWS.S3.PutObjectOutput,
ContentType: 'text/plain',
}
this.client.putObject(req, err => {
if (err) {
callback(false)
} else {
callback(true)
if (this.expire && !skipExpire) {
winston.warn('amazon s3 store cannot set expirations on keys')
}
}
})
}
}
export default AmazonS3DocumentStore

View file

@ -0,0 +1,22 @@
import type { Config } from '../../types/config'
import type { Store } from '../../types/store'
const build = async (config: Config): Promise<Store> => {
if (process.env.REDISTOGO_URL && config.storage.type === 'redis') {
// const redisClient = require("redis-url").connect(process.env.REDISTOGO_URL);
// Store = require("./lib/document-stores/redis");
// preferredStore = new Store(config.storage, redisClient);
const DocumentStore = (await import(`../document-stores/${config.storage.type}`)).default
return new DocumentStore(config.storage)
}
const DocumentStore = (await import(`../document-stores/${config.storage.type}`)).default
return new DocumentStore(config.storage)
}
export default build

View file

@ -0,0 +1,79 @@
import * as winston from 'winston'
import * as fs from 'fs'
import * as crypto from 'crypto'
import type { Callback, Store } from '../../types/store'
import type { FileStoreConfig } from '../../types/config'
// Generate md5 of a string
const md5 = (str: string) => {
const md5sum = crypto.createHash('md5')
md5sum.update(str)
return md5sum.digest('hex')
}
// For storing in files
// options[type] = file
// options[path] - Where to store
class FileDocumentStore implements Store {
type: string
expire?: number | undefined
basePath: string
constructor(options: FileStoreConfig) {
this.basePath = options.path || './data'
this.expire = options.expire
this.type = options.type
}
// Get data from a file from key
get = (
key: string,
callback: Callback,
skipExpire?: boolean | undefined,
): void => {
const fn = `${this.basePath}/${md5(key)}`
fs.readFile(fn, 'utf8', (err, data) => {
if (err) {
callback(false)
} else {
callback(data)
if (this.expire && !skipExpire) {
winston.warn('file store cannot set expirations on keys')
}
}
})
}
// Save data in a file, key as md5 - since we don't know what we could
// be passed here
set = (
key: string,
data: string,
callback: Callback,
skipExpire?: boolean | undefined,
): void => {
try {
fs.mkdir(this.basePath, '700', () => {
const fn = `${this.basePath}/${md5(key)}`
fs.writeFile(fn, data, 'utf8', err => {
if (err) {
callback(false)
} else {
callback(true)
if (this.expire && !skipExpire) {
winston.warn('file store cannot set expirations on keys')
}
}
})
})
} catch (err) {
callback(false)
}
}
}
export default FileDocumentStore

View file

@ -0,0 +1,114 @@
import { Datastore, PathType } from '@google-cloud/datastore'
import * as winston from 'winston'
import type { Callback, Store } from '../../types/store'
import type { GoogleStoreConfig } from '../../types/config'
class GoogleDatastoreDocumentStore implements Store {
kind: string
expire?: number
datastore: Datastore
type: string
// Create a new store with options
constructor(options: GoogleStoreConfig) {
this.kind = 'Haste'
this.expire = options.expire
this.type = options.type
this.datastore = new Datastore()
}
// Save file in a key
set = (
key: PathType,
data: string,
callback: Callback,
skipExpire?: boolean,
) => {
const expireTime =
skipExpire || this.expire === undefined
? null
: new Date(Date.now() + this.expire * 1000)
const taskKey = this.datastore.key([this.kind, key])
const task = {
key: taskKey,
data: [
{
name: 'value',
value: data,
excludeFromIndexes: true,
},
{
name: 'expiration',
value: expireTime,
},
],
}
this.datastore
.insert(task)
.then(() => {
callback(true)
})
.catch(() => {
callback(false)
})
}
// Get a file from a key
get = (key: PathType, callback: Callback, skipExpire?: boolean): void => {
const taskKey = this.datastore.key([this.kind, key])
this.datastore
.get(taskKey)
.then(entity => {
if (skipExpire || entity[0].expiration == null) {
callback(entity[0].value)
} else if (entity[0].expiration < new Date()) {
winston.info('document expired', {
key,
expiration: entity[0].expiration,
check: new Date(),
})
callback(false)
} else {
// update expiry
const task = {
key: taskKey,
data: [
{
name: 'value',
value: entity[0].value,
excludeFromIndexes: true,
},
{
name: 'expiration',
value: new Date(
Date.now() + (this.expire ? this.expire * 1000 : 0),
),
},
],
}
this.datastore
.update(task)
.then(() => {})
.catch(err => {
winston.error('failed to update expiration', { error: err })
})
callback(entity[0].value)
}
})
.catch(err => {
winston.error('Error retrieving value from Google Datastore', {
error: err,
})
callback(false)
})
}
}
export default GoogleDatastoreDocumentStore

View file

@ -0,0 +1,75 @@
import * as winston from 'winston'
import Memcached = require('memcached')
import type { Callback, Store } from '../../types/store'
import type { MemcachedStoreConfig } from '../../types/config'
class MemcachedDocumentStore implements Store {
expire: number | undefined
client?: Memcached
type: string
// Create a new store with options
constructor(options: MemcachedStoreConfig) {
this.expire = options.expire
this.type = options.type
const host = options.host || '127.0.0.1'
const port = options.port || 11211
const url = `${host}:${port}`
this.connect(url)
}
// Create a connection
connect = (url: string) => {
this.client = new Memcached(url)
winston.info(`connecting to memcached on ${url}`)
this.client.on('failure', (error: Memcached.IssueData) => {
winston.info('error connecting to memcached', { error })
})
}
// Get a file from a key
get = (
key: string,
callback: Callback,
skipExpire?: boolean | undefined,
): void => {
this.client?.get(key, (error, data: string) => {
const value = error ? false : data
callback(value as string)
// Update the key so that the expiration is pushed forward
if (value && !skipExpire) {
this.set(
key,
data,
updateSucceeded => {
if (!updateSucceeded) {
winston.error('failed to update expiration on GET', { key })
}
},
skipExpire,
)
}
})
}
// Save file in a key
set = (
key: string,
data: string,
callback: Callback,
skipExpire?: boolean | undefined,
): void => {
this.client?.set(key, data, skipExpire ? 0 : this.expire || 0, error => {
callback(!error)
})
}
}
export default MemcachedDocumentStore

View file

@ -0,0 +1,128 @@
import * as winston from 'winston'
import { MongoClient } from 'mongodb'
import type { Callback, Store } from '../../types/store'
import type { MongoStoreConfig } from '../../types/config'
type ConnectCallback = (error?: Error, db?: MongoClient) => void
class MongoDocumentStore implements Store {
type: string
expire?: number | undefined
connectionUrl: string
constructor(options: MongoStoreConfig) {
this.expire = options.expire
this.type = options.type
this.connectionUrl = process.env.DATABASE_URl || options.connectionUrl
}
safeConnect = (callback: ConnectCallback) => {
MongoClient.connect(this.connectionUrl, (err, client) => {
if (err) {
winston.error('error connecting to mongodb', { error: err })
callback(err)
} else {
callback(undefined, client)
}
})
}
get = (
key: string,
callback: Callback,
skipExpire?: boolean | undefined,
): void => {
const now = Math.floor(new Date().getTime() / 1000)
this.safeConnect((err, client) => {
if (err) return callback(false)
return client
?.db()
.collection('entries')
.findOne(
{
entry_id: key,
$or: [{ expiration: -1 }, { expiration: { $gt: now } }],
},
(error?: Error, entry?) => {
if (error) {
winston.error('error persisting value to mongodb', { error })
return callback(false)
}
callback(entry === null ? false : entry?.value)
if (
entry !== null &&
entry?.expiration !== -1 &&
this.expire &&
!skipExpire
) {
return client
.db()
.collection('entries')
.update(
{
entry_id: key,
},
{
$set: {
expiration: this.expire + now,
},
},
{},
() => {},
)
}
return true
},
)
})
}
set = (
key: string,
data: string,
callback: Callback,
skipExpire?: boolean | undefined,
): void => {
const now = Math.floor(new Date().getTime() / 1000)
this.safeConnect((err, client) => {
if (err) return callback(false)
return client
?.db()
.collection('entries')
.update(
{
entry_id: key,
$or: [{ expiration: -1 }, { expiration: { $gt: now } }],
},
{
entry_id: key,
value: data,
expiration: this.expire && !skipExpire ? this.expire + now : -1,
},
{
upsert: true,
},
(error?: Error) => {
if (error) {
winston.error('error persisting value to mongodb', { error })
return callback(false)
}
return callback(true)
},
)
})
}
}
export default MongoDocumentStore

View file

@ -0,0 +1,112 @@
import * as winston from 'winston'
import { Pool, PoolClient } from 'pg'
import type { Callback, Store } from '../../types/store'
import type { PostgresStoreConfig } from '../../types/config'
type ConnectCallback = (
error?: Error,
client?: PoolClient,
done?: () => void,
) => void
// A postgres document store
class PostgresDocumentStore implements Store {
type: string
expireJS?: number
pool: Pool
constructor(options: PostgresStoreConfig) {
this.expireJS = options.expire
this.type = options.type
const connectionString = process.env.DATABASE_URL || options.connectionUrl
this.pool = new Pool({ connectionString })
}
// A connection wrapper
safeConnect = (callback: ConnectCallback) => {
this.pool.connect((error: Error, client: PoolClient, done: () => void) => {
if (error) {
winston.error('error connecting to postgres', { error })
callback(error)
} else {
callback(undefined, client, done)
}
})
}
// Get a given key's data
get = (
key: string,
callback: Callback,
skipExpire?: boolean | undefined,
): void => {
const now = Math.floor(new Date().getTime() / 1000)
this.safeConnect((err, client, done): void => {
if (err) {
return callback(false)
}
return client?.query(
'SELECT id,value,expiration from entries where KEY = $1 and (expiration IS NULL or expiration > $2)',
[key, now],
(error: Error, result) => {
if (error) {
winston.error('error retrieving value from postgres', {
error,
})
return callback(false)
}
callback(result.rows.length ? result.rows[0].value : false)
if (result.rows.length && this.expireJS && !skipExpire) {
return client.query(
'UPDATE entries SET expiration = $1 WHERE ID = $2',
[this.expireJS + now, result.rows[0].id],
(currentErr: Error) => {
if (!currentErr) {
return done?.()
}
return callback(false)
},
)
}
return done?.()
},
)
})
}
// Set a given key
set = (
key: string,
data: string,
callback: Callback,
skipExpire?: boolean | undefined,
): void => {
const now = Math.floor(new Date().getTime() / 1000)
this.safeConnect((err, client, done) => {
if (err) {
return callback(false)
}
return client?.query(
'INSERT INTO entries (key, value, expiration) VALUES ($1, $2, $3)',
[key, data, this.expireJS && !skipExpire ? this.expireJS + now : null],
(error: Error) => {
if (error) {
winston.error('error persisting value to postgres', { error })
return callback(false)
}
callback(true)
return done?.()
},
)
})
}
}
export default PostgresDocumentStore

View file

@ -0,0 +1,58 @@
import * as winston from 'winston'
import * as crypto from 'crypto'
import rethink, { RethinkClient } from 'rethinkdbdash'
import type { RethinkDbStoreConfig } from '../../types/config'
import type { Callback } from '../../types/store'
const md5 = (str: string) => {
const md5sum = crypto.createHash('md5')
md5sum.update(str)
return md5sum.digest('hex')
}
class RethinkDBStore {
client: RethinkClient
constructor(options: RethinkDbStoreConfig) {
this.client = rethink({
silent: true,
host: options.host || '127.0.0.1',
port: options.port || 28015,
db: options.db || 'haste',
user: options.user || 'admin',
password: options.password || '',
})
}
set = (key: string, data: string, callback: Callback): void => {
this.client
.table('uploads')
.insert({ id: md5(key), data })
.run(error => {
if (error) {
callback(false)
winston.error('failed to insert to table', error)
return
}
callback(true)
})
}
get = (key: string, callback: Callback): void => {
this.client
.table('uploads')
.get(md5(key))
.run((error, result) => {
if (error || !result) {
callback(false)
if (error) winston.error('failed to insert to table', error)
return
}
callback(result.data)
})
}
}
export default RethinkDBStore

21
src/lib/helpers/config.ts Normal file
View file

@ -0,0 +1,21 @@
import * as fs from 'fs'
import { Config } from '../../types/config'
const getConfig = (): Config => {
const configPath = process.argv.length <= 2 ? 'config.json' : process.argv[2]
const config = JSON.parse(fs.readFileSync(configPath, 'utf8'))
config.port = (process.env.PORT || config.port || 7777) as number
config.host = process.env.HOST || config.host || 'localhost'
if (!config.storage) {
config.storage = { type: 'file' }
}
if (!config.storage.type) {
config.storage.type = 'file'
}
return config
}
export default getConfig

View file

@ -0,0 +1,7 @@
import * as path from 'path'
export const getStaticDirectory = (baseDirectory: string) =>
path.join(baseDirectory, '..', 'static')
export const getStaticItemDirectory = (baseDirectory: string, item: string) =>
path.join(baseDirectory, '..', 'static', item)

23
src/lib/helpers/log.ts Normal file
View file

@ -0,0 +1,23 @@
import * as winston from 'winston'
import type { Config } from '../../types/config'
const addLogging = (config: Config) => {
try {
winston.remove(winston.transports.Console)
} catch (e) {
/* was not present */
}
let detail
let type
for (let i = 0; i < config.logging.length; i += 1) {
detail = config.logging[i]
type = detail.type
const transport = winston.transports[type]
winston.add(transport, detail)
}
}
export default addLogging

View file

@ -0,0 +1,13 @@
import type { KeyGenerator } from '../../types/key-generator'
import type { Config } from '../../types/config'
const build = async (config: Config): Promise<KeyGenerator> => {
const pwOptions = config.keyGenerator
pwOptions.type = pwOptions.type || 'random'
const Generator = (await import(`../key-generators/${pwOptions.type}`)).default
const keyGenerator = new Generator(pwOptions)
return keyGenerator
}
export default build

View file

@ -0,0 +1,41 @@
import * as fs from 'fs'
import type { KeyGeneratorConfig } from '../../types/config'
import type { KeyGenerator } from '../../types/key-generator'
class DictionaryGenerator implements KeyGenerator {
type: string
dictionary: string[]
constructor(options: KeyGeneratorConfig, readyCallback: () => void) {
// Check options format
if (!options) throw Error('No options passed to generator')
if (!options.path) throw Error('No dictionary path specified in options')
this.dictionary = []
this.type = options.type
// Load dictionary
fs.readFile(options.path, 'utf8', (err, data) => {
if (err) throw err
this.dictionary = data.split(/[\n\r]+/)
if (readyCallback) readyCallback()
})
}
// Generates a dictionary-based key, of keyLength words
createKey(keyLength: number): string {
let text = ''
for (let i = 0; i < keyLength; i += 1) {
const index = Math.floor(Math.random() * this.dictionary.length)
text += this.dictionary[index]
}
return text
}
}
export default DictionaryGenerator

View file

@ -0,0 +1,34 @@
// Draws inspiration from pwgen and http://tools.arantius.com/password
import type { KeyGeneratorConfig } from '../../types/config'
import type { KeyGenerator } from '../../types/key-generator'
const randOf = (collection: string) => () =>
collection[Math.floor(Math.random() * collection.length)]
// Helper methods to get an random vowel or consonant
const randVowel = randOf('aeiou')
const randConsonant = randOf('bcdfghjklmnpqrstvwxyz')
class PhoneticKeyGenerator implements KeyGenerator {
type: string
constructor(options: KeyGeneratorConfig) {
this.type = options.type
}
// Generate a phonetic key of alternating consonant & vowel
// eslint-disable-next-line class-methods-use-this
createKey(keyLength: number) {
let text = ''
const start = Math.round(Math.random())
for (let i = 0; i < keyLength; i += 1) {
text += i % 2 === start ? randConsonant() : randVowel()
}
return text
}
}
export default PhoneticKeyGenerator

View file

@ -0,0 +1,30 @@
import type { KeyGeneratorConfig } from '../../types/config'
import type { KeyGenerator } from '../../types/key-generator'
class RandomKeyGenerator implements KeyGenerator {
type: string
keyspace: string
// Initialize a new generator with the given keySpace
constructor(options: KeyGeneratorConfig) {
this.keyspace =
options.keyspace ||
'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
this.type = options.type
}
// Generate a key of the given length
createKey(keyLength: number): string {
let text = ''
for (let i = 0; i < keyLength; i += 1) {
const index = Math.floor(Math.random() * this.keyspace.length)
text += this.keyspace.charAt(index)
}
return text
}
}
export default RandomKeyGenerator

8
src/server.ts Normal file
View file

@ -0,0 +1,8 @@
import winston = require('winston')
import App from './app'
const { server, config } = new App()
server.listen(config.port, config.host, () => {
winston.info(`listening on ${config.host}:${config.port}`)
})

68
src/types/config.ts Normal file
View file

@ -0,0 +1,68 @@
import { Logging } from './log'
import { RateLimits } from './rate-limits'
export interface Config {
host: string
port: number
keyLength: number
maxLength: number
staticMaxAge: number
recompressStaticAssets: boolean
logging: Logging[]
keyGenerator: KeyGeneratorConfig
rateLimits: RateLimits
storage: StoreConfig
documents: Record<string, string>
}
export type BaseStoreConfig = {
type: string
expire?: number
}
export interface MongoStoreConfig extends BaseStoreConfig {
connectionUrl: string
}
export interface MemcachedStoreConfig extends BaseStoreConfig {
host: string
port: number
}
export interface FileStoreConfig extends BaseStoreConfig {
path: string
}
export interface AmazonStoreConfig extends BaseStoreConfig {
bucket: string
region: string
}
export interface PostgresStoreConfig extends BaseStoreConfig {
connectionUrl: string
}
export interface RethinkDbStoreConfig extends BaseStoreConfig {
host: string
port: string
db: string
user: string
password: string
}
export type GoogleStoreConfig = BaseStoreConfig
export type StoreConfig =
| MongoStoreConfig
| MemcachedStoreConfig
| GoogleStoreConfig
| AmazonStoreConfig
| FileStoreConfig
| MongoStoreConfig
export interface KeyGeneratorConfig {
type: string
keyspace?: string
path?: string
}

15
src/types/document.ts Normal file
View file

@ -0,0 +1,15 @@
import type { Config } from './config'
import type { KeyGenerator } from './key-generator'
import type { Store } from './store'
export type Document = {
store: Store
config: Config
maxLength: number
keyLength: number
keyGenerator: KeyGenerator
}
export interface Documents {
about: string
}

View file

@ -0,0 +1,4 @@
export interface KeyGenerator {
type: string
createKey?: (a: number) => string
}

12
src/types/log.ts Normal file
View file

@ -0,0 +1,12 @@
export interface Logging {
level: string
type:
| 'File'
| 'Console'
| 'Loggly'
| 'DailyRotateFile'
| 'Http'
| 'Memory'
| 'Webhook'
}

13
src/types/rate-limits.ts Normal file
View file

@ -0,0 +1,13 @@
export interface Normal {
totalRequests: number
every: number
}
export interface Categories {
normal: Normal
}
export interface RateLimits {
end?: boolean
categories: Categories
}

5
src/types/request.ts Normal file
View file

@ -0,0 +1,5 @@
import DocumentHandler from '../lib/document-handler'
export type RequestParams = {
documentHandler: DocumentHandler
}

13
src/types/store.ts Normal file
View file

@ -0,0 +1,13 @@
export type Callback = (arg0: boolean | string) => void
export interface Store {
type: string
expire?: number
get: (key: string, callback: Callback, skipExpire?: boolean) => void
set: (
key: string,
data: string,
callback: Callback,
skipExpire?: boolean,
) => void
}

39
tsconfig.json Normal file
View file

@ -0,0 +1,39 @@
{
"ts-node": {
"files": true
},
"files": ["src/global.d.ts"],
"compilerOptions": {
"allowJs": true,
"composite": false,
"declaration": true,
"declarationMap": true,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"incremental": true,
"inlineSources": false,
"isolatedModules": true,
"lib": ["ES2021", "DOM", "DOM.Iterable"],
"moduleResolution": "node",
"noUnusedLocals": false,
"noUnusedParameters": false,
"preserveWatchOutput": true,
"resolveJsonModule": true,
"skipLibCheck": true,
"strict": true,
"typeRoots": ["node_modules/@types", "src/global.d.ts"],
"target": "es6",
"noEmit": false,
"module": "commonjs",
"sourceMap": true,
"rootDir": ".",
"outDir": "dist",
// "baseUrl": "./src",
"paths": {
// "~/*": ["/src/*"],
// "~/lib/*": ["./src/lib/*"]
}
},
"include": ["src", "global.d.ts", "**/*.ts"],
"exclude": ["node_modules"]
}

5
tsconfig.lint.json Normal file
View file

@ -0,0 +1,5 @@
{
"extends": "./tsconfig.json",
"include": ["src"],
"exclude": ["node_modules", "dist", "coverage"]
}

4175
yarn-error.log Normal file

File diff suppressed because it is too large Load diff

4076
yarn.lock Normal file

File diff suppressed because it is too large Load diff