Merge pull request #1518 from SkynetLabs/nocache-large-files

prevent files larger than 1GB from being stored in cache

docker/nginx/conf.d/include/location-skylink

@@ -65,6 +65,12 @@ access_by_lua_block {
         return ngx.exit(ngx.status)
     end
 
+    -- if skylink is found on nocache list then set internal nocache variable
+    -- to tell nginx that it should not try and cache this file (too large)
+    if ngx.shared.nocache:get(ngx.var.skylink_v1) then
+        ngx.var.nocache = "1"
+    end
+
     -- this block runs only when accounts are enabled
     if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end
 
@@ -95,6 +101,11 @@ header_filter_by_lua_block {
     if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then
         ngx.header["Skynet-Proof"] = ngx.var.skynet_proof
     end
+
+    -- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold
+    if tonumber(ngx.header["Content-Length"]) > 1e+9 then
+        ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"])
+    end
 }
 
 limit_rate_after 512k;

docker/nginx/conf.d/include/proxy-cache-downloads

@@ -1,7 +1,14 @@
-set $nocache 0; # internal variable for bypassing the cache, nginx expects 0/1 for boolean
 proxy_cache skynet; # cache name
 proxy_cache_key $skylink_v1$path$arg_format$arg_attachment$arg_start$arg_end$http_range; # unique cache key
 proxy_cache_min_uses 3; # cache after 3 uses
 proxy_cache_valid 200 206 307 308 48h; # keep 200, 206, 307 and 308 responses valid for up to 2 days
-proxy_cache_bypass $nocache $cookie_nocache $arg_nocache; # add cache bypass option
 add_header X-Proxy-Cache $upstream_cache_status; # add response header to indicate cache hits and misses
+
+# bypass - this will bypass cache hit on request (status BYPASS)
+# but still stores file in cache if cache conditions are met
+proxy_cache_bypass $cookie_nocache $arg_nocache;
+
+# no cache - this will ignore cache on request (status MISS)
+# and does not store file in cache under no condition
+set_if_empty $nocache "0";
+proxy_no_cache $nocache;

docker/nginx/nginx.conf

@@ -77,6 +77,10 @@ http {
     # that gives us capacity of around 100k entries in 30 megabyte dictionary
     lua_shared_dict blocklist 30m;
 
+    # create a shared dictionary to fill with skylinks that should not
+    # be cached due to the large size or some other reasons
+    lua_shared_dict nocache 10m;
+
     # this runs before forking out nginx worker processes
     init_by_lua_block { 
         require "cjson"