From 7fab576e6adb787932a278a085144b71eb94fc50 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Thu, 29 Apr 2021 16:56:53 +0200 Subject: [PATCH] atomic writes --- docker-compose.yml | 134 +----------------- docker/nginx/conf.d/include/cors | 10 +- .../src/services/authServerSideProps.js | 2 +- packages/health-check/package.json | 2 + .../src/adapters/FileSyncAtomic.js | 28 ++++ packages/health-check/src/db.js | 6 +- yarn.lock | 4 +- 7 files changed, 42 insertions(+), 144 deletions(-) create mode 100644 packages/health-check/src/adapters/FileSyncAtomic.js diff --git a/docker-compose.yml b/docker-compose.yml index 65a30ed6..cc0e35a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,135 +14,6 @@ networks: - subnet: 10.10.10.0/24 services: - sia: - build: - context: ./docker/sia - dockerfile: Dockerfile - args: - branch: v1.5.5 - container_name: sia - restart: unless-stopped - logging: *default-logging - environment: - - SIA_MODULES=gctwr - env_file: - - .env - volumes: - - ./docker/data/sia:/sia-data - networks: - shared: - ipv4_address: 10.10.10.10 - expose: - - 9980 - - caddy: - build: - context: ./docker/caddy - dockerfile: Dockerfile - container_name: caddy - restart: unless-stopped - logging: *default-logging - env_file: - - .env - volumes: - - ./docker/data/caddy/data:/data - - ./docker/data/caddy/config:/config - - ./docker/caddy/Caddyfile:/etc/caddy/Caddyfile - networks: - shared: - ipv4_address: 10.10.10.20 - ports: - - "80:80" - - "443:443" - depends_on: - - nginx - - nginx: - build: - context: ./docker/nginx - dockerfile: Dockerfile - container_name: nginx - restart: unless-stopped - logging: *default-logging - env_file: - - .env - volumes: - - ./docker/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro - - ./docker/nginx/conf.d:/etc/nginx/conf.d:ro - - ./docker/data/nginx/cache:/data/nginx/cache - - ./docker/data/nginx/logs:/usr/local/openresty/nginx/logs - - ./docker/data/nginx/skynet:/data/nginx/skynet:ro - - ./docker/data/sia/apipassword:/data/sia/apipassword:ro - networks: - shared: - ipv4_address: 10.10.10.30 - expose: - - 80 - depends_on: - - sia - - health-check - - handshake-api - - website - - website: - build: - context: ./packages/website - dockerfile: Dockerfile - container_name: website - restart: unless-stopped - logging: *default-logging - env_file: - - .env - networks: - shared: - ipv4_address: 10.10.10.35 - expose: - - 9000 - - handshake: - build: - context: ./docker/handshake - dockerfile: Dockerfile - container_name: handshake - restart: unless-stopped - logging: *default-logging - environment: - - HSD_LOG_CONSOLE=false - - HSD_HTTP_HOST=0.0.0.0 - - HSD_NETWORK=main - - HSD_PORT=12037 - env_file: - - .env - volumes: - - ./docker/data/handshake/.hsd:/root/.hsd - networks: - shared: - ipv4_address: 10.10.10.40 - expose: - - 12037 - - handshake-api: - build: - context: ./packages/handshake-api - dockerfile: Dockerfile - container_name: handshake-api - restart: unless-stopped - logging: *default-logging - environment: - - HOSTNAME=0.0.0.0 - - HSD_HOST=handshake - - HSD_NETWORK=main - - HSD_PORT=12037 - env_file: - - .env - networks: - shared: - ipv4_address: 10.10.10.50 - expose: - - 3100 - depends_on: - - handshake - health-check: build: context: ./packages/health-check @@ -157,10 +28,7 @@ services: ipv4_address: 10.10.10.60 environment: - HOSTNAME=0.0.0.0 - - PORTAL_URL=http://nginx + - PORTAL_URL=https://siasky.net - STATE_DIR=/usr/app/state expose: - 3100 - depends_on: - - handshake - - handshake-api diff --git a/docker/nginx/conf.d/include/cors b/docker/nginx/conf.d/include/cors index 61075e52..7a969986 100644 --- a/docker/nginx/conf.d/include/cors +++ b/docker/nginx/conf.d/include/cors @@ -1,8 +1,8 @@ if ($request_method = 'OPTIONS') { more_set_headers 'Access-Control-Allow-Origin: $http_origin'; more_set_headers 'Access-Control-Allow-Credentials: true'; - more_set_headers 'Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE'; - more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + more_set_headers 'Access-Control-Allow-Methods: GET, POST, HEAD, OPTIONS, PUT, PATH, DELETE'; + more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,X-HTTP-Method-Override,upload-offset,upload-length,tus-version,tus-resumable,tus-extension,tus-max-size'; more_set_headers 'Access-Control-Max-Age: 1728000'; more_set_headers 'Content-Type: text/plain; charset=utf-8'; more_set_headers 'Content-Length: 0'; @@ -11,6 +11,6 @@ if ($request_method = 'OPTIONS') { more_set_headers 'Access-Control-Allow-Origin: $http_origin'; more_set_headers 'Access-Control-Allow-Credentials: true'; -more_set_headers 'Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE'; -more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; -more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range,Skynet-File-Metadata,Skynet-Skylink,Skynet-Portal-Api'; +more_set_headers 'Access-Control-Allow-Methods: GET, POST, HEAD, OPTIONS, PUT, PATH, DELETE'; +more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,X-HTTP-Method-Override,upload-offset,upload-length,tus-version,tus-resumable,tus-extension,tus-max-size'; +more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range,Skynet-File-Metadata,Skynet-Skylink,Skynet-Portal-Api,upload-offset,upload-length,tus-version,tus-resumable,tus-extension,tus-max-size'; diff --git a/packages/dashboard/src/services/authServerSideProps.js b/packages/dashboard/src/services/authServerSideProps.js index bc7bf43c..71cec614 100644 --- a/packages/dashboard/src/services/authServerSideProps.js +++ b/packages/dashboard/src/services/authServerSideProps.js @@ -1,6 +1,6 @@ import ky from "ky/umd"; -const isProduction = process.env.NODE_ENV === "production"; +const isProduction = false; // process.env.NODE_ENV === "production"; export default function authServerSideProps(getServerSideProps) { return function authenticate(context) { diff --git a/packages/health-check/package.json b/packages/health-check/package.json index cb25fd12..468860b9 100644 --- a/packages/health-check/package.json +++ b/packages/health-check/package.json @@ -8,10 +8,12 @@ "express": "^4.17.1", "form-data": "^3.0.1", "got": "^11.8.2", + "graceful-fs": "^4.2.6", "hasha": "^5.2.2", "http-status-codes": "^2.1.2", "lodash": "^4.17.21", "lowdb": "^1.0.0", + "write-file-atomic": "^3.0.3", "yargs": "^16.2.0" }, "devDependencies": { diff --git a/packages/health-check/src/adapters/FileSyncAtomic.js b/packages/health-check/src/adapters/FileSyncAtomic.js new file mode 100644 index 00000000..247481e9 --- /dev/null +++ b/packages/health-check/src/adapters/FileSyncAtomic.js @@ -0,0 +1,28 @@ +const fs = require("graceful-fs"); +const Base = require("lowdb/adapters/Base"); +const { sync: writeFileAtomicSync } = require("write-file-atomic"); + +class FileSyncAtomic extends Base { + read() { + if (fs.existsSync(this.source)) { + try { + const data = fs.readFileSync(this.source, "utf-8").trim(); + return data ? this.deserialize(data) : this.defaultValue; + } catch (e) { + if (e instanceof SyntaxError) { + e.message = `Malformed JSON in file: ${this.source}\n${e.message}`; + } + throw e; + } + } else { + writeFileAtomicSync(this.source, this.serialize(this.defaultValue)); + return this.defaultValue; + } + } + + write(data) { + return writeFileAtomicSync(this.source, this.serialize(data)); + } +} + +module.exports = FileSyncAtomic; diff --git a/packages/health-check/src/db.js b/packages/health-check/src/db.js index 6dca215c..d7e47708 100644 --- a/packages/health-check/src/db.js +++ b/packages/health-check/src/db.js @@ -1,10 +1,10 @@ -const fs = require("fs"); +const fs = require("graceful-fs"); const low = require("lowdb"); -const FileSync = require("lowdb/adapters/FileSync"); +const FileSyncAtomic = require("./adapters/FileSyncAtomic"); if (!fs.existsSync(process.env.STATE_DIR)) fs.mkdirSync(process.env.STATE_DIR); -const adapter = new FileSync(`${process.env.STATE_DIR}/state.json`); +const adapter = new FileSyncAtomic(`${process.env.STATE_DIR}/state.json`); const db = low(adapter); db.defaults({ disabled: false, critical: [], extended: [] }).write(); diff --git a/yarn.lock b/yarn.lock index fb9dc094..6ca94bfc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8428,7 +8428,7 @@ got@^9.6.0: to-readable-stream "^1.0.0" url-parse-lax "^3.0.0" -graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.3, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.2, graceful-fs@^4.2.3, graceful-fs@^4.2.4: +graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.3, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.2, graceful-fs@^4.2.3, graceful-fs@^4.2.4, graceful-fs@^4.2.6: version "4.2.6" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee" integrity sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ== @@ -17627,7 +17627,7 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= -write-file-atomic@^3.0.0: +write-file-atomic@^3.0.0, write-file-atomic@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8" integrity sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==