OpenHarmony/third_party_mesa3d
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

doc/ci: Update nginx caching snippets

Browse Source 
Fix the nginx cache snippets - I'd missed the file nesting somehow.
Tested on a debian:bookworm image with nginx-full installed, checked
that we could pull an arbitrary external site, as well as S3, as well
as GitLab artifacts.

Signed-off-by: Daniel Stone <daniels@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34341>
This commit is contained in:
Daniel Stone
2025-04-02 11:17:00 +01:00
committed by Marge Bot
parent 6393ebbdbb
commit 7c73b9a498
2 changed files with 101 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
docs/ci/fdo-cache
View File

@@ -1,4 +1,4 @@
proxy_cache_path /var/cache/nginx/ levels=1:2 keys_zone=my_cache:10m max_size=24g inactive=48h use_temp_path=off;
proxy_cache_path /var/cache/nginx/ levels=1:2 keys_zone=my_cache:10m max_size=50g inactive=2w use_temp_path=off;
server {
listen 10.42.0.1:80 default_server;
@@ -13,32 +13,80 @@ server {
server_name _;
add_header X-GG-Cache-Status $upstream_cache_status;
proxy_cache my_cache;
location / {
# First attempt to serve request as file, then
# as directory, then fall back to displaying a 404.
try_files $uri $uri/ =404;
}
location /cache_gitlab_artifacts {
internal;
# Gitlabs http server puts everything as no-cache even though
# the artifacts URLS don't change. So enforce a long validity
# time and ignore the headers that defeat caching
proxy_cache_valid 200 48h;
proxy_ignore_headers Cache-Control Set-Cookie;
include snippets/uri-caching.conf;
}
location /tmp {
# Lava server http artifacts to the clients; e.g. for the deploy action
alias /var/lib/lava/dispatcher/tmp;
}
location /cache {
# special case gitlab artifacts
if ($arg_uri ~* /.*gitlab.*artifacts(\/|%2F)raw/ ) {
rewrite ^ /cache_gitlab_artifacts;
}
# Set a really low validity together with cache revalidation; Our goal
# for caching isn't to lower the number of http requests but to
# lower the amount of data transfer. Also for some test
# scenarios (typical manual tests) the file at a given url
# might get modified so avoid confusion by ensuring
# revalidations happens often.
proxy_cache_valid 200 10s;
proxy_cache_revalidate on;
include snippets/uri-caching.conf;
}
proxy_cache my_cache;
# Wait for the cache creation when multiple query are done for the same file
proxy_cache_lock on;
proxy_cache_lock_age 30m;
proxy_cache_lock_timeout 1h;
location /force_cache {
internal;
# On some setups the cache headers will indicate to nginx that the
# artifacts shouldn't be cached, however if we know that that is not valid
# for lava usage this endpoint allows caching to be forced instead
proxy_cache_valid 200 48h;
proxy_ignore_headers Cache-Control Set-Cookie expires;
include snippets/uri-caching.conf;
}
location /fdo_cache {
internal;
# As the auth information in the query is being dropped, use
# the minimal possible cache validity, such that in practise
# every requests gets revalidated. This avoids
# unauthenticated downloads from our cache as the cache key doesn't
# include auth info
proxy_cache_valid 200 1s;
proxy_cache_revalidate on;
proxy_ignore_headers Cache-Control Set-Cookie expires;
set_by_lua_block $cache_key {
-- Set the cache key to the uri with the query stripped
local unescaped = ngx.unescape_uri(ngx.var.arg_uri);
local it,err = ngx.re.match(unescaped, "([^?]*).*")
if not it then
-- Fallback on the full uri as key if the regexp fails
return ngx.var.arg_uri;
end
return it[1]
}
proxy_cache_key $cache_key;
include snippets/uri-caching.conf;
}
location /cache {
# Gitlabs http server puts everything as no-cache even though
# the artifacts URLS don't change.
if ($arg_uri ~* /.*gitlab.*artifacts(\/|%2F)raw/ ) {
rewrite ^ /force_cache;
}
# fd.o's object storage has an embedded signature for
# authentication as part of its query. So use an adjusted cache key
# without the query
if ($arg_uri ~* .*your-objectstorage.com(\/|%2F)fdo-opa(\/|%2F)) {
rewrite ^ /fdo_cache;
}
# Set a really low validity together with cache revalidation; Our goal
# for caching isn't to lower the number of http requests but to
# lower the amount of data transfer. Also for some test
# scenarios (typical manual tests) the file at a given url
# might get modified so avoid confusion by ensuring
# revalidations happens often.
proxy_cache_valid 200 10s;
proxy_cache_revalidate on;
include snippets/uri-caching.conf;
}
}

139
docs/ci/uri-caching.conf
View File

@@ -1,131 +1,44 @@
set $proxy_authorization '';
set_by_lua $proxyuri '
local unescaped = ngx.unescape_uri(ngx.var.arg_uri);
local it, err = ngx.re.match(unescaped, "(https?://)(.*@)?([^/]*)(/.*)?");
if not it then
-- Hack to cause nginx to return 404
return "http://localhost/404"
end
local unescaped = ngx.unescape_uri(ngx.var.arg_uri);
local it, err = ngx.re.match(unescaped, "(https?://)(.*@)?([^/]*)(/.*)?");
if not it then
-- Hack to cause nginx to return 404
return "http://localhost/404"
end
local scheme = it[1];
local authstring = it[2];
local host = it[3];
local query = it[4];
local scheme = it[1];
local authstring = it[2];
local host = it[3];
local query = it[4];
if ngx.var.http_authorization and ngx.var.http_authorization ~= "" then
ngx.var.proxy_authorization = ngx.var.http_authorization;
elseif authstring then
auth = string.sub(authstring, 0, -2);
auth64 = ngx.encode_base64(auth);
ngx.var.proxy_authorization = "Basic " .. auth64;
end
if ngx.var.http_authorization and ngx.var.http_authorization ~= "" then
ngx.var.proxy_authorization = ngx.var.http_authorization;
elseif authstring then
auth = string.sub(authstring, 0, -2);
auth64 = ngx.encode_base64(auth);
ngx.var.proxy_authorization = "Basic " .. auth64;
end
-- Default to / if none is set to avoid using the request_uri query
if not query then
query = "/";
end
-- Default to / if none is set to avoid using the request_uri query
if not query then
query = "/";
end
return scheme .. host .. query;
return scheme .. host .. query;
';
# Rewrite the location header to redirect back to this server. Do
# this using lua header filtering to allow for url encoding the original
# location header for use as a query parameter.
header_filter_by_lua_block {
if ngx.header.location then
ngx.header.location = "/cache?uri=" .. ngx.escape_uri(ngx.header.location);
end
if ngx.header.location then
ngx.header.location = "/cache?uri=" .. ngx.escape_uri(ngx.header.location);
end
}
add_header X-GG-Cache-Status $upstream_cache_status;
proxy_set_header Authorization $proxy_authorization;
proxy_pass $proxyuri;
proxy_cache_path /var/cache/nginx/ levels=1:2 keys_zone=my_cache:10m max_size=50g inactive=2w use_temp_path=off;
server {
listen 80 default_server;
listen [::]:80 default_server;
root /var/www/html;
server_name _;
location / {
# First attempt to serve request as file, then
# as directory, then fall back to displaying a 404.
try_files $uri $uri/ =404;
}
add_header X-GG-Cache-Status $upstream_cache_status;
proxy_cache my_cache;
# dnsmasq should be running on all dispatchers; listening on 127.0.0.1/8 for
# those running in docker 127.0.0.11 is the default resolver so this address
# covers both use-cases
resolver 127.0.0.11;
# Wait for the cache creation when multiple query are done for the same file
proxy_cache_lock on;
proxy_cache_lock_age 30m;
proxy_cache_lock_timeout 1h;
location /force_cache {
internal;
# On some setups the cache headers will indicate to nginx that the
# artifacts shouldn't be cached, however if we know that that is not valid
# for lava usage this endpoint allows caching to be forced instead
proxy_cache_valid 200 48h;
proxy_ignore_headers Cache-Control Set-Cookie expires;
include snippets/uri-caching.conf;
}
location /fdo_cache {
internal;
# As the auth information in the query is being dropped, use
# the minimal possible cache validity, such that in practise
# every requests gets revalidated. This avoids
# unauthenticated downloads from our cache as the cache key doesn't
# include auth info
proxy_cache_valid 200 1s;
proxy_cache_revalidate on;
proxy_ignore_headers Cache-Control Set-Cookie expires;
set_by_lua_block $cache_key {
-- Set the cache key to the uri with the query stripped
local unescaped = ngx.unescape_uri(ngx.var.arg_uri);
local it,err = ngx.re.match(unescaped, "([^?]*).*")
if not it then
-- Fallback on the full uri as key if the regexp fails
return ngx.var.arg_uri;
end
return it[1]
}
proxy_cache_key $cache_key;
include snippets/uri-caching.conf;
}
location /cache {
# Gitlabs http server puts everything as no-cache even though
# the artifacts URLS don't change.
if ($arg_uri ~* /.*gitlab.*artifacts(\/|%2F)raw/ ) {
rewrite ^ /force_cache;
}
# fd.o's object storage has an embedded signature for
# authentication as part of its query. So use an adjusted cache key
# without the query
if ($arg_uri ~* .*your-objectstorage.com(\/|%2F)fdo-opa(\/|%2F)) {
rewrite ^ /fdo_cache;
}
# Set a really low validity together with cache revalidation; Our goal
# for caching isn't to lower the number of http requests but to
# lower the amount of data transfer. Also for some test
# scenarios (typical manual tests) the file at a given url
# might get modified so avoid confusion by ensuring
# revalidations happens often.
proxy_cache_valid 200 10s;
proxy_cache_revalidate on;
include snippets/uri-caching.conf;
}
}