Dev1 Galaxy Forum / HOWTO: package caching with nginx

Re: HOWTO: package caching with nginx

dummy@example.com (specing) — Wed, 20 Feb 2019 20:59:50 +0000

I have updated the config, my initial one missed deb.debian.org (see amprolla rewrites).

Re: HOWTO: package caching with nginx

dummy@example.com (specing) — Tue, 19 Feb 2019 16:52:24 +0000

Everything hitting mirror.local/devuan goes to the reverse proxied server (in this case ftp.fau.de), including /devuan/merged.

Re: HOWTO: package caching with nginx

dummy@example.com (KatolaZ) — Wed, 13 Feb 2019 22:17:03 +0000

specing wrote:

Put this config inside the http section:

#proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_cache_path /data/nginx/devuan levels=2 keys_zone=devuan-cache:4m max_size=10000m inactive=1y loader_files=1000 loader_threshold=10000;
# caching options
proxy_temp_path  /data/nginx/disttmp;

# TODO: proxy_ssl_verify!
server {
    listen 80;
    server_name mirror.local;

    access_log /var/log/nginx/proxy_mirror.access_log main;
    error_log  /tmp/proxy_mirror.error_log debug;

    access_log on;
    error_log on;

    location /devuan {
        proxy_pass https://ftp.fau.de/devuan;
        proxy_set_header Host ftp.fau.de;
        proxy_redirect default;

        proxy_cache devuan-cache;
        # for items with response codes 200 (Success:OK) and 302 (Redirection:Found)
        proxy_cache_valid 200 302 3y;
        # for 404 (ClientError:Not Found)
        proxy_cache_valid 404 1m;

        proxy_cache_revalidate on;
    }
}

IMHO, this should not work on its own, since you still need a good deal of redirecting. You still need to provide paths and redirects for /merged (having just /devuan will not be enough) and those have to be redirected to either a devuan or a debian mirror, as needed. But I might be wrong.

KatolaZ

HOWTO: package caching with nginx

dummy@example.com (specing) — Wed, 13 Feb 2019 20:53:54 +0000

Put this config inside the http section and modify it as you see fit. I recommend changing back-end mirrors (ftp.fau.de and debian.ethz.ch), cache size (max_size) and cache paths (/data/nginx/*).

#proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_cache_path /data/nginx/devuan levels=2 keys_zone=devuan-cache:4m max_size=10000m inactive=1y loader_files=1000 loader_threshold=10000;
proxy_cache_path /data/nginx/debian levels=2 keys_zone=debian-cache:4m max_size=10000m inactive=1y loader_files=1000 loader_threshold=10000;
proxy_cache_path /data/nginx/debian-security levels=2 keys_zone=debian-security-cache:4m max_size=10000m inactive=1y loader_files=1000 loader_threshold=10000;
# caching options
proxy_temp_path  /data/nginx/disttmp;

# TODO: proxy_ssl_verify!
server {
	listen 80;
	server_name proxy.local;

	access_log /var/log/nginx/proxy_mirror.access_log main;
	error_log  /tmp/proxy_mirror.error_log debug;

	access_log on;
	error_log on;

	location /devuan {
		proxy_pass https://ftp.fau.de/devuan;
		proxy_set_header Host ftp.fau.de;
		proxy_redirect default;
		proxy_redirect https://ftp.fau.de /;
		# see https://git.devuan.org/devuan-infrastructure/amprolla3/blob/master/contrib/nginx.conf#L12
		proxy_redirect http://auto.mirror.devuan.org/ /;
		proxy_redirect https://auto.mirror.devuan.org/ /;
		proxy_redirect http://deb.debian.org /;
		proxy_redirect https://deb.debian.org /;

		proxy_cache devuan-cache;
		# for items with response codes 200 (Success:OK) and 302 (Redirection:Found)
		proxy_cache_valid 200 302 3y;
		# for 404 (ClientError:Not Found)
		proxy_cache_valid 404 1m;

		proxy_cache_revalidate on;
	}

	location /debian {
		proxy_pass https://debian.ethz.ch/debian;
		proxy_set_header Host debian.ethz.ch;
		proxy_redirect default;
		proxy_redirect https://debian.ethz.ch /;

		proxy_cache debian-cache;
		# for items with response codes 200 (Success:OK) and 302 (Redirection:Found)
		proxy_cache_valid 200 302 3y;
		# for 404 (ClientError:Not Found)
		proxy_cache_valid 404 1m;

		proxy_cache_revalidate on;
	}

	location /debian-security {
		proxy_pass https://debian.ethz.ch/debian-security;
		proxy_set_header Host debian.ethz.ch;
		proxy_redirect default;
		proxy_redirect https://debian.ethz.ch /;

		proxy_cache debian-security-cache;
		# for items with response codes 200 (Success:OK) and 302 (Redirection:Found)
		proxy_cache_valid 200 302 3y;
		# for 404 (ClientError:Not Found)
		proxy_cache_valid 404 1m;

		proxy_cache_revalidate on;
	}
}

In "Configure the package manager":
- select "enter information manually" on the archive mirror country
- use proxy.local as hostname
- use /devuan/merged as path
- if installer says that the specified mirror does not support your architecture, then that probably means that the proxy server cannot reach the real server (in the case of long timeout) or that it is forwarding a different directory (short timeout)