diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 93c5fe2..d39a825 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,6 +14,33 @@ jobs: - name: Validate hoprd.cfg.yaml.tpl run: ./test/validate-hoprd-config.sh + - name: Check node's entrypoint and key storage settings + run: ./test/entrypoint.sh + + - name: Check that gnosisvpn-server claims its fixed address + run: ./test/fixed-address.sh + + - name: Check that VPN clients can't reach local resources + run: ./test/firewall.sh + + - name: Check the blocklist and per-peer shaping + run: ./test/traffic.sh + + # Same checks inside the image that ships, so a missing bash/iproute2/iptables/ipset/curl + # fails here rather than on a DAppNode. Pulls the same base image the package build below + # does. + - name: Build the gnosisvpn-server image for the image checks + run: docker build -t gnosisvpn-server:checks gnosisvpn-server + + - name: Check the firewall inside the built gnosisvpn-server image + run: FIREWALL_TEST_IMAGE=gnosisvpn-server:checks ./test/firewall.sh + + - name: Check the blocklist and shaping inside the built gnosisvpn-server image + run: TRAFFIC_TEST_IMAGE=gnosisvpn-server:checks ./test/traffic.sh + + - name: Check the fixed address with the built gnosisvpn-server image + run: FIXED_ADDRESS_TEST_IMAGE=gnosisvpn-server:checks ./test/fixed-address.sh + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/Dockerfile b/Dockerfile index da87848..5ac4ace 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,5 @@ ARG UPSTREAM_VERSION -# hoprd-pix-test, not hoprd: the plain image enables no PIX pool feature, so it inherits -# hopr-lib's default `pix-bjj` and announces BabyJubJub as its PIX curve suite. -# Change back to `hoprd` once Curvy pool is ready -FROM europe-west3-docker.pkg.dev/hoprassociation/docker-images/hoprd-pix-test:${UPSTREAM_VERSION} +FROM europe-west3-docker.pkg.dev/hoprassociation/docker-images/hoprd:${UPSTREAM_VERSION} # not used at the moment, but might be useful in the future ENV DAPPNODE=true @@ -12,4 +9,5 @@ ADD entrypoint.sh /bin/entrypoint.sh RUN chmod +rx /bin/entrypoint.sh -ENTRYPOINT ["/bin/entrypoint.sh"] +# tini (shipped in the upstream image) stays PID 1 so SIGTERM reaches hoprd on stop/update. +ENTRYPOINT ["/bin/tini", "--", "/bin/entrypoint.sh"] diff --git a/README.md b/README.md index d2f4359..29e0fd7 100644 --- a/README.md +++ b/README.md @@ -12,15 +12,134 @@ Runs a [GnosisVPN](https://vpn.gnosis.eth.limo) exit node on DAppNode: a HOPR no same `dncore_network` every DAppNode package service gets attached to (a non-core package can't get `network_mode: "service:node"` here either - DAppNode's `dappmanager` forces that `networks:` attachment onto every service regardless, and Docker Compose rejects - `network_mode` and `networks` together on the same service). Since `hoprd`'s - `session_ip_forwarding` needs a literal `ip:port` (hostnames are rejected by hoprd's own - config parser) and a non-core package can't pin a static IP on `dncore_network` either (see - below), `node`'s `entrypoint.sh` resolves `gnosisvpn-server`'s DAppNode DNS alias - (`gnosisvpn-server..dappnode`) to its current IP at boot and renders that into - `hoprd.cfg.yaml.tpl` before `hoprd` starts (see `GNOSISVPN_SERVER_HOST` in - `docker-compose.yml`). Unlike the loopback address this replaced, this IP is **not** fixed - across installs, and isn't guaranteed stable across a `gnosisvpn-server` recreate - see - "Known limitations" below. + `network_mode` and `networks` together on the same service). `node` reaches it at + **`172.33.255.254`** (`:51820` WireGuard, `:8000` control API) - see "Fixed address". + +## Fixed address + +`gnosisvpn-server`'s address is `172.33.255.254` on every install and across recreates. + +DAppNode doesn't let a package pin a container IP: `dappmanager` strips `ipv4_address` and +only allows its own networks. So +[`gvpn-fixed-address.sh`](gnosisvpn-server/gvpn-fixed-address.sh) adds the address to +`gnosisvpn-server`'s `dncore_network` interface itself. It is inside `dncore_network`'s +`172.33.0.0/16`, so `node` reaches it directly. The script: + +- refuses the address if anything else already answers for it, +- announces it, so `node` stops using a previous container's MAC, +- claims it again within 30s if a network reconnect drops it. + +Limits: Docker doesn't know about the address, but it hands out the lowest free one first, so +it only reaches `172.33.255.254` on a nearly full network. If DAppNode retires +`dncore_network`, the address must move to `dnprivate_network` (`10.20.0.0/24`). + +## Firewall + +`gnosisvpn-server` has its own network namespace, and +[`gnosisvpn-server/wg-hooks.sh`](gnosisvpn-server/wg-hooks.sh) sets its forwarding policy. +`wg-quick` runs the script from `PostUp`/`PreDown` in `wggvpn.conf`. It is a port of the +`wg-hooks.sh` + `iptables.rules` pair from the exit-node appliance image (`gnosis_vpn-exit_os`). +The appliance version can't be used as-is, because a DAppNode package never sees the host's +network namespace (see "Known limitations" below). + +VPN clients are untrusted. Without the firewall, their traffic can reach the DAppNode host, the +dappmanager, every package on `dncore_network` and the LAN behind the host. With it, only +public destinations are reachable: + +- FORWARD policy is DROP. Client traffic goes through `GVPN_PRIVATE` first, which drops: + - RFC1918, CGNAT, link-local, loopback, `0.0.0.0/8` and multicast/reserved + - `dncore_network` (`172.33.0.0/16`), which is not in RFC1918 + - the subnet this container is attached to +- Only `wggvpn -> ` is accepted. Return traffic is `RELATED,ESTABLISHED` only, so the + exit node can't open connections to clients. +- INPUT from `wggvpn` accepts only the ICMP echo that the GnosisVPN client sends to check the + tunnel (`connection.ping`, default `10.128.0.1`). The control API on port 8000 and all other + local services are not reachable from the tunnel. +- `up` removes its old rules first, so a restart doesn't add duplicates. `down` leaves FORWARD + at DROP. +- `entrypoint.sh` sets FORWARD to DROP before it starts `gnosis_vpn-server`. `wg-quick` brings + `wggvpn` up before `PostUp` runs the hook, so this closes that gap. + +Not covered: + +- IPv6. `gnosis_vpn-server` adds peers with IPv4 `allowed-ips` only and `wggvpn` has no IPv6 + address, so WireGuard drops IPv6 client packets. The appliance also drops IPv6 with + `ip6tables`; this package does not. Re-check this when `gnosis_vpn-server` gets IPv6 support + ([gnosis/gnosis_vpn-client#330](https://github.com/gnosis/gnosis_vpn-client/issues/330)). +- The DAppNode's own public IPv4 address. It is a public destination, so it passes. (The + appliance finds and blocks it with an external HTTPS lookup; this package doesn't.) What a + client reaches there depends on where the address lives: + - On the host's NIC (a VPS, or a public IP from the ISP): every service the host listens on + and every published port, e.g. SSH and the hoprd API on `3001`. Don't run this package on + such a host unless the host firewall restricts those services by source. + - On a router in front of the host: whatever the router hairpins back, e.g. its admin UI if + it listens on WAN, and every forwarded port. Assume NAT loopback is on. +`just test-firewall` and `just test-firewall-image` check these rules (see +[`test/README.md`](test/README.md)). + +## Blocklist + +[`gnosisvpn-server/gvpn-blocklist-update.sh`](gnosisvpn-server/gvpn-blocklist-update.sh) fills +the `gvpn-blocklist` ipset, and `wg-hooks.sh` drops traffic to it, and traffic from it back to +clients, before the client ACCEPT. Ported from the appliance, which uses the same FireHOL level1 +feed: + +- `entrypoint.sh` loads the cached list before it starts the server, so the last known-good list + filters the first forwarded packet. The cache lives in the `gnosisvpn-data` volume. +- It then runs the refresher as a child process. A container has no timer, so the appliance's + twice-daily systemd timer becomes "refresh now, then every 12 h plus up to 1 h of jitter". +- A refresh swaps the whole set in atomically (`ipset swap`), so forwarding never sees a + half-built list. +- It fails open. An unreachable feed, or one that parses to fewer than 1000 or more than 131072 + entries, leaves the previous list in place and logs why. A poisoned `0.0.0.0/1` can't + blackhole every session, because prefixes shorter than /8 are only accepted for the 224/3 + bogon range. +- IPv4 only, like the isolation rules - see "Not covered" above for why client IPv6 never + reaches the tunnel. + +Feeds are configured from the DAppNode UI, see "Settings in the DAppNode UI" below. + +## Per-peer shaping + +[`gnosisvpn-server/gvpn-peer-shaping.sh`](gnosisvpn-server/gvpn-peer-shaping.sh) builds an HTB +tree on `wggvpn` when `wg-hooks.sh` runs, with one class per peer slot: + +- Download (server to client) is classified by destination address on `wggvpn` egress. +- Upload (client to internet) is mirrored from `wggvpn` ingress onto an `ifb-gvpn` device and + classified there by source address. It has to happen on the `wggvpn` side: the uplink only + ever sees post-MASQUERADE addresses. +- Each peer is capped at `GVPN_PEER_RATE_KBIT` (default 10000) and all peers together at + `GVPN_PEER_SHAPING_AGGREGATE_KBIT` (default 25000). It queues rather than drops, so TCP + settles at the cap instead of sawtoothing, and fq_codel signals with delay before loss + because a dropped packet costs an end-to-end retransmit across the mixnet. +- `entrypoint.sh` derives the peer range from `WIREGUARD_CIDR` and `MAX_SLOTS`, the same way it + derives the pool for `config.toml`, and exports it for the `wg-quick` process that runs the + hook. +- Unlike the appliance, a shaping failure does not stop the exit node. `tc` and the `ifb`/`sch_*` + modules come from the DAppNode host's kernel, and this container can't load modules into it. A + host without them serves unshaped and logs `per-peer shaping failed` on every interface sync. + +`just test-traffic` and `just test-traffic-image` check both of these (see +[`test/README.md`](test/README.md)). + +## Settings in the DAppNode UI + +The exit node's limits and blocklist are optional fields in the package's setup wizard, so they +are set in the DAppNode UI (the package's **Config**, or the wizard on install) rather than by +editing `docker-compose.yml`. Leaving a field empty keeps the default: + +| UI field | Variable | Default | +|---|---|---| +| Per-client bandwidth limit | `GVPN_PEER_RATE_KBIT` | `10000` kbit/s | +| Total bandwidth limit for all clients | `GVPN_PEER_SHAPING_AGGREGATE_KBIT` | `25000` kbit/s | +| Blocklist feeds | `GVPN_BLOCKLIST_FEEDS` | FireHOL level1 | +| Blocklist refresh interval | `GVPN_BLOCKLIST_REFRESH_INTERVAL_S` | `43200` s | +| WireGuard client subnet | `WIREGUARD_CIDR` | `10.128.0.0/24` | +| Maximum connected peers | `MAX_SLOTS` | `4` | + +Feed URLs are space-separated; one per line works too. The rest of the `GVPN_BLOCKLIST_*` +variables (minimum entries, download timeout, jitter) stay environment-only, with the +appliance's defaults. ## Known limitations vs. production @@ -46,19 +165,10 @@ explicit firewall rule. For the same reason, this exit node's internal `gnosisvpn-server` address is **not** the fixed `172.30.0.1` a production, Ansible-managed exit node presents (a fixed docker bridge gateway, identical across that whole fleet because every host pins the same subnet - -something this package structurally can't do). Worse, unlike a first attempt at working -around this (sharing `node`'s network namespace to get a fixed loopback address - see git -history), the address DAppNode assigns `gnosisvpn-server` on `dncore_network` is **not fixed -across installs either**: it's a normal dynamic bridge IP, different on every install and not -guaranteed stable across a `gnosisvpn-server` recreate on the same install (though it survives -an ordinary restart). `entrypoint.sh` logs it on every `node` boot -(`gnosisvpn-server resolved to ...`), so it's discoverable per-install from DAppNode's log -viewer, but there's currently no single documented value a `gnosis_vpn-client` can be -pre-configured with the way `172.30.0.1` works for the production fleet - a client would need -to be told this specific exit node's current address out of band. The proper fix is upstream, -in `hoprd`: an in-progress change to have `hoprd` inform a connecting client of the correct -target itself, so clients don't need this hardcoded at all. Until that lands, this is a real -gap for this package's exit nodes specifically. +something this package structurally can't do). It is `172.33.255.254` instead, so a +`gnosis_vpn-client` needs its `[connection.bridge]`/`[connection.wg]` `target` set to +`172.33.255.254:8000`/`172.33.255.254:51820`. That setting is global to the client. The proper +fix is upstream: `hoprd` telling a connecting client the target itself. ## Local testing diff --git a/dappnode_package.json b/dappnode_package.json index 38fe366..0b78045 100644 --- a/dappnode_package.json +++ b/dappnode_package.json @@ -1,7 +1,7 @@ { "name": "gnosisvpn-exit-node.public.dappnode.eth", "version": "1.0.0", - "upstreamVersion": "4.0.3", + "upstreamVersion": "5.0.3", "description": "Run a GnosisVPN exit node: a HOPR node (hoprd) that relays privacy-preserving traffic through the HOPR mixnet, paired with gnosis_vpn-server, which terminates that traffic on a local WireGuard interface and forwards it to the internet. gnosis_vpn-server's own release is tracked separately (see README) since it isn't the SDK-tracked upstream.", "type": "service", "architectures": [ diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 02f1a65..b42c560 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -11,7 +11,12 @@ services: HOPRD_PASSWORD: ${HOPRD_PASSWORD} HOPRD_API_TOKEN: ${HOPRD_API_TOKEN} HOPRD_HOST: ${HOPRD_HOST:-127.0.0.1:9091} - GNOSISVPN_SERVER_HOST: gnosisvpn-server gnosisvpn-server: environment: WIREGUARD_CIDR: ${WIREGUARD_CIDR:-10.128.0.0/24} +networks: + # dncore_network's subnet, which gnosisvpn-server's fixed address 172.33.255.254 is in. + default: + ipam: + config: + - subnet: 172.33.0.0/16 diff --git a/docker-compose.yml b/docker-compose.yml index a74199f..d66ab73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: context: . args: # see https://console.cloud.google.com/artifacts/docker/hoprassociation/europe-west3/docker-images/hoprd - UPSTREAM_VERSION: 5.0.0-rc.1-commit.7d90171 + UPSTREAM_VERSION: 5.0.3 ports: - "1422:1422/udp" - "1422:1422/tcp" @@ -21,7 +21,8 @@ services: - "seccomp:unconfined" environment: RUST_LOG: info - HOPRD_CONFIGURATION_FILE_PATH: /app/hoprd/conf/hoprd.cfg.yaml + # Where entrypoint.sh renders the config. Not the upload path hoprd.custom.cfg.yaml. + HOPRD_CONFIGURATION_FILE_PATH: /app/hoprd/conf/hoprd.generated.cfg.yaml HOPRD_IDENTITY: /app/hoprd/conf/hopr.id HOPRD_DATA: /app/hoprd/data HOPRD_HOST: "127.0.0.1:9091" @@ -40,19 +41,13 @@ services: HOPRD_NAT: "true" ADDITIONAL_CMDLINE_ARGS: "" ADDITIONAL_ENVIRONMENT_VARS: "" - # DAppNode's dappmanager forces every service (this one included) onto dncore_network - # with a DNS alias of ..dappnode - see getShortUniqueDnp in - # DNP_DAPPMANAGER. hoprd's session_ip_forwarding needs a literal ip:port (hostnames are - # rejected by hoprd's own config parser), so entrypoint.sh resolves this at boot and - # renders it into hoprd.cfg.yaml.tpl - see entrypoint.sh and README. - GNOSISVPN_SERVER_HOST: "gnosisvpn-server.gnosisvpn-exit-node.public.dappnode" restart: unless-stopped - depends_on: - - gnosisvpn-server deploy: resources: limits: - memory: 2G + memory: 8GB + depends_on: + - gnosisvpn-server gnosisvpn-server: # Do not set image field here, it will be set automatically by the sdk. build: @@ -65,6 +60,13 @@ services: WIREGUARD_CIDR: "10.128.0.0/24" WIREGUARD_MTU: "1420" MAX_SLOTS: "4" + # Per-peer traffic shaping, in kbit/s: each client is capped at the per-peer rate and + # all clients together at the aggregate. See README's "Per-peer shaping". + GVPN_PEER_RATE_KBIT: "10000" + GVPN_PEER_SHAPING_AGGREGATE_KBIT: "25000" + # Space-separated threat feeds for the blocklist. The last known-good list is cached + # in the gnosisvpn-data volume. See README's "Blocklist". + GVPN_BLOCKLIST_FEEDS: "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset" restart: unless-stopped deploy: resources: diff --git a/entrypoint.sh b/entrypoint.sh index 9ca40e3..6be4c19 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -8,48 +8,96 @@ if [ -z "${HOPRD_BLOKLI_URL:-}" ]; then unset HOPRD_BLOKLI_URL fi +PLACEHOLDER=__GNOSISVPN_SERVER_IP__ +# gnosisvpn-server claims this address itself; must match ADDRESS in +# gnosisvpn-server/gvpn-fixed-address.sh. +GNOSISVPN_SERVER_IP=172.33.255.254 +BLOKLI_PLACEHOLDER=__BLOKLI_URL__ +# The setup wizard's upload. Only read, never written: older versions rendered here, and the +# next boot took that output for an upload. CUSTOM_CONFIG=/app/hoprd/conf/hoprd.custom.cfg.yaml TEMPLATE_CONFIG=/app/hoprd.cfg.yaml.tpl -RENDERED_CONFIG="${HOPRD_CONFIGURATION_FILE_PATH:?HOPRD_CONFIGURATION_FILE_PATH must be set}" +# What hoprd reads. Hardcoded so it can't be pointed back at CUSTOM_CONFIG. +GENERATED_CONFIG=/app/hoprd/conf/hoprd.generated.cfg.yaml + +# This image has no sed. The `echo x` / `%x` keeps the trailing newline `$(cat)` strips. +read_config() { + local contents + contents="$(cat "$1"; echo x)" + printf '%s' "${contents%x}" +} + +# A config without comments and blank lines. +config_body() { + local line body="" + while IFS= read -r line; do + line="${line%%#*}" + [ -z "${line//[[:space:]]/}" ] && continue + body+="${line}"$'\n' + done <<<"$1" + printf '%s' "${body}" +} + +# Whether a config is an older version's output: every template they rendered has this +# WireGuard entry, which they filled in with gnosisvpn-server's dynamic address. +rendered_by_older_version() { + local line wg_entry="^ *- \"([0-9]{1,3}(\\.[0-9]{1,3}){3}):51820\" +# gnosisvpn-server's WireGuard interface" + while IFS= read -r line; do + if [[ "${line}" =~ ${wg_entry} ]] && [ "${BASH_REMATCH[1]}" != "${GNOSISVPN_SERVER_IP}" ]; then + return 0 + fi + done <<<"$1" + return 1 +} + +# A config's top-level blokli_url. +config_blokli_url() { + local line + while IFS= read -r line; do + if [[ "${line}" =~ ^blokli_url:[[:space:]]*\"?([^\"#[:space:]]+) ]]; then + printf '%s' "${BASH_REMATCH[1]}" + return + fi + done <<<"$1" +} + +template_contents="$(read_config "${TEMPLATE_CONFIG}")" +source_contents="${template_contents}" if [ -s "${CUSTOM_CONFIG}" ]; then - # A config was uploaded via the setup wizard's "Custom HOPR node configuration file" - # field - use it as-is (modulo the same __GNOSISVPN_SERVER_IP__ substitution below), so we - # never clobber a deliberate override otherwise. - echo "entrypoint.sh: using uploaded ${CUSTOM_CONFIG}" - SOURCE_CONFIG="${CUSTOM_CONFIG}" -else - SOURCE_CONFIG="${TEMPLATE_CONFIG}" + custom_contents="$(read_config "${CUSTOM_CONFIG}")" + if rendered_by_older_version "${custom_contents}"; then + # Using it would keep its stale address and miss newer template settings. + echo "entrypoint.sh: ignoring ${CUSTOM_CONFIG}: an older version of this package rendered it. Upload your config again if it was a custom one." >&2 + else + echo "entrypoint.sh: using uploaded ${CUSTOM_CONFIG}" + source_contents="${custom_contents}" + custom_body="$(config_body "${custom_contents}")" + case "${custom_body}" in + *"${PLACEHOLDER}"* | *"${GNOSISVPN_SERVER_IP}"*) ;; + *) + echo "entrypoint.sh: WARNING: ${CUSTOM_CONFIG} does not forward to gnosisvpn-server at ${GNOSISVPN_SERVER_IP}; add ${GNOSISVPN_SERVER_IP}:51820 and ${GNOSISVPN_SERVER_IP}:8000 to session_ip_forwarding.target_allow_list." >&2 + ;; + esac + case "${custom_body}" in + *"${BLOKLI_PLACEHOLDER}"*) ;; + *) + echo "entrypoint.sh: WARNING: ${CUSTOM_CONFIG} does not set the Pix strategy's pool.blokli_url to ${BLOKLI_PLACEHOLDER}, so PIX payments may not work; see hoprd.cfg.yaml.tpl." >&2 + ;; + esac + fi fi -: "${GNOSISVPN_SERVER_HOST:?GNOSISVPN_SERVER_HOST must be set}" -echo "entrypoint.sh: resolving gnosisvpn-server (${GNOSISVPN_SERVER_HOST})..." -gnosisvpn_server_ip="" -for _ in $(seq 1 30); do - gnosisvpn_server_ip="$(dig +short "${GNOSISVPN_SERVER_HOST}" A | head -n1)" - [ -n "${gnosisvpn_server_ip}" ] && break - sleep 1 -done -if [ -z "${gnosisvpn_server_ip}" ]; then - echo "entrypoint.sh: FATAL: could not resolve ${GNOSISVPN_SERVER_HOST} after 30s - is gnosisvpn-server running?" >&2 +# hoprd prefers HOPRD_BLOKLI_URL to the config's URL; the PIX pool only reads the config. +# A URL set through ADDITIONAL_ENVIRONMENT_VARS or ADDITIONAL_CMDLINE_ARGS reaches only hoprd. +blokli_url="${HOPRD_BLOKLI_URL:-$(config_blokli_url "${source_contents}")}" +if [ -z "${blokli_url}" ] && [[ "$(config_body "${source_contents}")" == *"${BLOKLI_PLACEHOLDER}"* ]]; then + echo "entrypoint.sh: no blokli_url for the PIX pool: write the top-level blokli_url as \"blokli_url: \" on one line, or set HOPRD_BLOKLI_URL." >&2 exit 1 fi -# Logged deliberately: this is currently the only way to find gnosisvpn-server's address, -# which a connecting gnosis_vpn-client needs as its session target - see README. hoprd is -# expected to gain a way to tell clients this directly (tracked upstream), making this -# per-install manual lookup unnecessary. -echo "entrypoint.sh: gnosisvpn-server resolved to ${gnosisvpn_server_ip}" - -# __GNOSISVPN_SERVER_IP__ is substituted the same way regardless of which file above won - -# see setup-wizard.yml's "Custom HOPR node configuration file" field, which documents using -# the same placeholder in an uploaded config too, instead of requiring the uploader to -# hardcode an address that isn't fixed and that hoprd can't resolve itself. -# No `sed` in this image (it's a minimal coreutils/util-linux build, no busybox either) - do -# the substitution with bash builtins instead. The `; echo x` / `%x` dance preserves a -# trailing newline that a bare `$(cat ...)` would otherwise strip. -config_contents="$(cat "${SOURCE_CONFIG}"; echo x)" -config_contents="${config_contents%x}" -printf '%s' "${config_contents//__GNOSISVPN_SERVER_IP__/${gnosisvpn_server_ip}}" >"${RENDERED_CONFIG}" -export HOPRD_CONFIGURATION_FILE_PATH="${RENDERED_CONFIG}" - -env ${ADDITIONAL_ENVIRONMENT_VARS:-} /bin/docker-entrypoint.sh ${ADDITIONAL_CMDLINE_ARGS:-} +rendered="${source_contents//${PLACEHOLDER}/${GNOSISVPN_SERVER_IP}}" +printf '%s' "${rendered//${BLOKLI_PLACEHOLDER}/"${blokli_url}"}" >"${GENERATED_CONFIG}" +export HOPRD_CONFIGURATION_FILE_PATH="${GENERATED_CONFIG}" + +# exec, so tini's SIGTERM reaches hoprd. +exec env ${ADDITIONAL_ENVIRONMENT_VARS:-} /bin/docker-entrypoint.sh ${ADDITIONAL_CMDLINE_ARGS:-} diff --git a/gnosisvpn-server/Dockerfile b/gnosisvpn-server/Dockerfile index b4858d7..9977d47 100644 --- a/gnosisvpn-server/Dockerfile +++ b/gnosisvpn-server/Dockerfile @@ -1,17 +1,18 @@ FROM europe-west3-docker.pkg.dev/gnosisvpn-production/docker-images/gnosis_vpn-server:0.7.0@sha256:443c4264bb41a82526ea7d227676f5060633e489569b4fb7fe6299f6a28ff78d -# The base image (Alpine) ships wg/wg-quick/ip but not iptables/ipset, which the PostUp/PreDown -# rules in wggvpn.conf.tpl need (ported from the gnosisvpn-server Ansible role). -RUN apk add --no-cache iptables ip6tables ipset bash +# What the scripts need; apk skips any the base image (Alpine) already has. +RUN apk add --no-cache bash iproute2 iptables ipset curl iputils-arping WORKDIR /app # Replace the image's own config.toml/wggvpn.conf with our templated versions. entrypoint.sh # renders these and execs gnosis_vpn-server directly (not via the image's own wrapper.sh, # which would leave it as a child process instead of PID 1 - see entrypoint.sh for why that -# matters for shutdown). +# matters for shutdown). PostUp/PreDown in wggvpn.conf run wg-hooks.sh, the forwarding +# policy - see README's "Firewall" section. wg-hooks.sh runs gvpn-peer-shaping.sh, and +# entrypoint.sh runs gvpn-blocklist-update.sh and gvpn-fixed-address.sh. COPY config.toml.tpl wggvpn.conf.tpl ./ -COPY entrypoint.sh ./entrypoint.sh -RUN chmod +x ./entrypoint.sh +COPY entrypoint.sh wg-hooks.sh gvpn-blocklist-update.sh gvpn-peer-shaping.sh gvpn-fixed-address.sh ./ +RUN chmod +x ./entrypoint.sh ./wg-hooks.sh ./gvpn-blocklist-update.sh ./gvpn-peer-shaping.sh ./gvpn-fixed-address.sh ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/gnosisvpn-server/entrypoint.sh b/gnosisvpn-server/entrypoint.sh index bb55ec5..ae8af80 100755 --- a/gnosisvpn-server/entrypoint.sh +++ b/gnosisvpn-server/entrypoint.sh @@ -15,6 +15,19 @@ wg_address="${o1}.${o2}.${o3}.$((o4 + 1))" wg_start="${o1}.${o2}.${o3}.$((o4 + 2))" wg_end="${o1}.${o2}.${o3}.$((o4 + 1 + max_slots))" +# The pool has to stay inside the CIDR's last /24: per-peer shaping steers packets by +# the last octet only, and .255 is the broadcast address. +if (( o4 + 1 + max_slots > 254 )); then + echo "gnosisvpn-server: FATAL: WIREGUARD_CIDR ${WIREGUARD_CIDR} with MAX_SLOTS=${max_slots} needs last octets .$((o4 + 1))-.$((o4 + 1 + max_slots)), past .254" >&2 + exit 1 +fi + +# Consumed by wg-hooks.sh through gvpn-peer-shaping.sh, which wg-quick runs with this +# process's environment (gnosis_vpn-server spawns wg-quick as a plain child process). +export GVPN_PEER_SUBNET_PREFIX="${o1}.${o2}.${o3}" +export GVPN_PEER_MIN_OCTET=$((o4 + 2)) +export GVPN_PEER_MAX_OCTET=$((o4 + 1 + max_slots)) + # gnosis_vpn-server's own wrapper.sh generates a WireGuard key when PRIVATE_KEY isn't set, # but doesn't persist it - a fresh key on every restart would break peer registration. # Persist it in the /data volume instead, generating it once here. @@ -53,6 +66,23 @@ if [ "${current_ip_forward}" != "1" ]; then echo "gnosisvpn-server: WARNING: net.ipv4.ip_forward is '${current_ip_forward}', not '1', and this container cannot set it itself. VPN client traffic will not reach the internet until it's enabled on the DAppNode host." >&2 fi +# wg-quick brings wggvpn up (with its peers) before PostUp runs wg-hooks.sh, so close +# FORWARD now; wg-hooks.sh then opens only the public path. If this fails, the container +# fails: the exit node must not start without it. +iptables -P FORWARD DROP + +# The blocklist set lives in this container's namespace, so its cache lives in the /data +# volume and the refresh runs as a child process: a container has no timer. Loading the +# cache first (no network) filters the first forwarded packet with the last known-good +# list; the refresher then replaces it from the feeds in the background, failing open. +/app/gvpn-blocklist-update.sh --from-cache +/app/gvpn-blocklist-update.sh --loop & + +# The fixed address node forwards to. Without it nothing reaches the server, so a failed +# claim stops the container. +/app/gvpn-fixed-address.sh +/app/gvpn-fixed-address.sh --loop & + # Run the binary directly as PID 1 (not via the image's wrapper.sh, which launches it as a # child process) so Docker's SIGTERM on `stop`/`down` reaches gnosis_vpn-server itself. It # has its own graceful-shutdown handling, which is what actually runs wg-quick down/PreDown. diff --git a/gnosisvpn-server/gvpn-blocklist-update.sh b/gnosisvpn-server/gvpn-blocklist-update.sh new file mode 100755 index 0000000..6119c2d --- /dev/null +++ b/gnosisvpn-server/gvpn-blocklist-update.sh @@ -0,0 +1,253 @@ +#!/bin/bash +# Populates the blocklist ipset the FORWARD DROP rules in wg-hooks.sh use. +# Ported from gvpn-exit-os (itself from gnosis_vpn-infrastructure, +# roles/gnosisvpn-blocklist): the Ansible variables arrive here as environment +# variables instead of exit.env. +# +# Three modes: +# --from-cache load the local cache file only (no network). entrypoint.sh +# runs this before gnosis_vpn-server, so the last known-good +# list is in effect for the first forwarded packet. +# --refresh fetch the threat feeds, refresh the cache, swap into the set. +# --loop refresh now, then every REFRESH_INTERVAL_S plus jitter. The +# appliance uses a systemd timer for this; a container has no +# timer, so entrypoint.sh runs this mode as a child process. +# +# Populating the live set always goes through an atomic `ipset swap`, so in-flight +# packets never see a partially-built set. The set is fail-open: any degraded +# path keeps the existing set (or an empty one) and the script still exits 0, +# reporting the problem in the log rather than failing the exit node. +set -euo pipefail + +IPSET_NAME="${GVPN_BLOCKLIST_IPSET_NAME:-gvpn-blocklist}" +IPSET_STAGING="${IPSET_NAME}-staging-$$" +IPSET_MAXELEM="${GVPN_BLOCKLIST_MAXELEM:-131072}" +CACHE_DIR="${GVPN_BLOCKLIST_CACHE_DIR:-/data/blocklist}" +CACHE_FILE="${CACHE_DIR}/blocklist.cidr" +MIN_ENTRIES="${GVPN_BLOCKLIST_MIN_ENTRIES:-1000}" +CURL_TIMEOUT="${GVPN_BLOCKLIST_CURL_TIMEOUT:-30}" +# Hard cap on each feed's download size. Bounds disk/memory use if a feed is +# compromised or misbehaving and streams an unexpectedly large body. +MAX_FILESIZE="${GVPN_BLOCKLIST_MAX_FILESIZE:-67108864}" +REFRESH_INTERVAL_S="${GVPN_BLOCKLIST_REFRESH_INTERVAL_S:-43200}" +REFRESH_JITTER_S="${GVPN_BLOCKLIST_REFRESH_JITTER_S:-3600}" +# Space-separated feed URLs. +FEEDS="${GVPN_BLOCKLIST_FEEDS:-https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset}" + +# Space- or newline-separated, so a value entered with one URL per line works +# too: `read` on its own would keep only the first line. +read -r -a FEED_LIST <<<"${FEEDS//$'\n'/ }" + +TMP_DIR="" +RESTORE_FILE="" + +usage() { + echo "usage: ${0} --from-cache|--refresh|--loop" >&2 + exit 64 +} + +log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] gvpn-blocklist: $*"; } + +cleanup() { + [[ -n "${RESTORE_FILE}" ]] && rm -f "${RESTORE_FILE}" + [[ -n "${TMP_DIR}" ]] && rm -rf "${TMP_DIR}" + ipset destroy "${IPSET_STAGING}" 2>/dev/null || true +} +trap cleanup EXIT + +# Strip comments (; and #), drop whitespace, keep only valid IPv4 / IPv4-CIDR +# tokens (bare IPs are valid hash:net members, treated as /32). +# +# IPv4-only by design: the live set is `family inet` and this parser rejects any +# non-IPv4 token. That is safe only while client IPv6 never enters the tunnel, +# which holds because gnosis_vpn-server adds every peer with an IPv4 /32 +# allowed-ips and wggvpn carries no IPv6 address. If that ever changes, blocked +# destinations become reachable over IPv6 and a family inet6 set is needed. +# +# The octet (0-255) and prefix (/0../32) sub-patterns are validated here so that +# ipset never sees an out-of-range token that would abort the whole restore. +# Prefixes shorter than /8 are accepted only for the statically dropped 224/3 +# bogon range: a single 0.0.0.0/1 or 128.0.0.0/1 would send every VPN session +# into the blocklist DROP while the entry-count checks stay green. +parse_feed() { + sed -e 's/[;#].*$//' -e 's/[[:space:]]//g' \ + | grep -E '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(/(3[0-2]|[12][0-9]|[0-9]))?$' \ + | awk -F '[./]' 'NF == 4 || $5 >= 8 || $0 == "224.0.0.0/3"' || true +} + +# Whether a parsed entry count is within the acceptable [MIN_ENTRIES, IPSET_MAXELEM] +# range. A count outside this band is treated as a degraded/broken list and the +# caller keeps the existing set instead of swapping (fail-safe). +count_within_bounds() { + local count="$1" + if [[ "${count}" -lt "${MIN_ENTRIES}" ]]; then + log "ERROR ${count} entries (< ${MIN_ENTRIES}); keeping existing set" + return 1 + fi + if [[ "${count}" -gt "${IPSET_MAXELEM}" ]]; then + log "ERROR ${count} entries (> maxelem ${IPSET_MAXELEM}); keeping existing set" + return 1 + fi + return 0 +} + +# Ensure the live set exists so the iptables DROP rules always resolve, even when +# we have nothing to populate it with. +ensure_live_set() { + ipset create "${IPSET_NAME}" hash:net family inet maxelem "${IPSET_MAXELEM}" -exist +} + +# Build and validate the staging set from $1. This is intentionally separated +# from cache publication so malformed feed/cache data never becomes the new +# known-good cache. +stage_set() { + local src="$1" + ipset destroy "${IPSET_STAGING}" 2>/dev/null || true + RESTORE_FILE=$(mktemp) || return 1 + { + echo "create ${IPSET_STAGING} hash:net family inet maxelem ${IPSET_MAXELEM}" + sed "s|^|add ${IPSET_STAGING} |" "${src}" + } >"${RESTORE_FILE}" || return 1 + # -exist makes the adds idempotent: a duplicate CIDR (e.g. a bare IP from one + # feed colliding with the same X/32 from another) no longer aborts the whole + # restore. + ipset restore -exist <"${RESTORE_FILE}" || return 1 + rm -f "${RESTORE_FILE}" + RESTORE_FILE="" +} + +# Atomically replace the live set's contents with the already-validated CIDRs in +# $1. +swap_in() { + local src="$1" + stage_set "${src}" || return 1 + ensure_live_set || return 1 + ipset swap "${IPSET_STAGING}" "${IPSET_NAME}" || return 1 + ipset destroy "${IPSET_STAGING}" 2>/dev/null || true +} + +publish_cache() { + local src="$1" + mkdir -p "${CACHE_DIR}" || return 1 + cp "${src}" "${CACHE_FILE}.tmp" || return 1 + mv "${CACHE_FILE}.tmp" "${CACHE_FILE}" || return 1 +} + +# Entries currently in the live set, for the log. +live_entries() { + ipset list "${IPSET_NAME}" 2>/dev/null | awk '/Number of entries/ { print $4 }' +} + +# Log what the live set holds after a degraded run: the set is fail-open, so a +# failed refresh keeps the previous list and shows up here only. +report_live_set() { + local entries + entries="$(live_entries || true)" + log "live set holds ${entries:-unknown} entries" +} + +populate_from_cache() { + local count + if [[ ! -s "${CACHE_FILE}" ]]; then + log "ERROR cache file ${CACHE_FILE} missing or empty; leaving set empty" + ensure_live_set || log "ERROR could not create ${IPSET_NAME}" + report_live_set + return 0 + fi + count=$(wc -l <"${CACHE_FILE}") + if ! count_within_bounds "${count}"; then + report_live_set + return 0 + fi + if ! swap_in "${CACHE_FILE}"; then + log "ERROR cache failed ipset validation; keeping existing set" + report_live_set + return 0 + fi + log "populated from cache: ${count} entries" +} + +refresh() { + local all url feed_failed=0 feed_success=0 count + # --loop calls this repeatedly and never reaches the EXIT trap, so each run + # clears the previous run's directory. + [[ -n "${TMP_DIR}" ]] && rm -rf "${TMP_DIR}" + TMP_DIR=$(mktemp -d) + all="${TMP_DIR}/all.cidr" + : >"${all}" + for url in "${FEED_LIST[@]}"; do + if curl -fsSL --max-time "${CURL_TIMEOUT}" --max-filesize "${MAX_FILESIZE}" --retry 3 --retry-delay 5 "${url}" >"${TMP_DIR}/feed"; then + parse_feed <"${TMP_DIR}/feed" >>"${all}" + feed_success=$((feed_success + 1)) + log "fetched ${url}" + else + feed_failed=1 + log "WARN failed to fetch ${url}" + fi + done + + if [[ "${feed_success}" -eq 0 ]]; then + log "ERROR all feeds failed; falling back to cache" + populate_from_cache + return 0 + fi + if [[ "${feed_failed}" -ne 0 ]]; then + log "ERROR one or more feeds failed; keeping existing set and cache" + report_live_set + return 0 + fi + + sort -u "${all}" -o "${all}" + count=$(wc -l <"${all}") + if ! count_within_bounds "${count}"; then + report_live_set + return 0 + fi + + if ! swap_in "${all}"; then + log "ERROR parsed CIDR list failed ipset validation; keeping existing set and cache" + report_live_set + return 0 + fi + if ! publish_cache "${all}"; then + log "ERROR failed to update cache after live swap" + report_live_set + return 0 + fi + log "refresh complete: ${count} entries" +} + +loop() { + local delay interval="${REFRESH_INTERVAL_S}" jitter="${REFRESH_JITTER_S}" + # A value that is not a number makes `sleep` fail, which would turn this into + # a busy refresh loop against the feeds. Fall back to the defaults instead. + if ! [[ "${interval}" =~ ^[1-9][0-9]*$ ]]; then + log "WARN GVPN_BLOCKLIST_REFRESH_INTERVAL_S='${interval}' is not a positive integer; using 43200" + interval=43200 + fi + if ! [[ "${jitter}" =~ ^[0-9]+$ ]]; then + log "WARN GVPN_BLOCKLIST_REFRESH_JITTER_S='${jitter}' is not a whole number; using 3600" + jitter=3600 + fi + while :; do + refresh || true + delay=$((interval + RANDOM % (jitter + 1))) + log "next refresh in ${delay}s" + sleep "${delay}" + done +} + +main() { + case "${1:-}" in + --from-cache) populate_from_cache ;; + --refresh) refresh ;; + --loop) loop ;; + *) usage ;; + esac +} + +# Only dispatch when executed directly, not when sourced (the check scripts +# exercise parse_feed and count_within_bounds in isolation). +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/gnosisvpn-server/gvpn-fixed-address.sh b/gnosisvpn-server/gvpn-fixed-address.sh new file mode 100755 index 0000000..a9c3bcc --- /dev/null +++ b/gnosisvpn-server/gvpn-fixed-address.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Claims gnosisvpn-server's fixed address on dncore_network (`gvpn-fixed-address.sh [--loop]`). +# +# DAppNode doesn't let a package pin a container IP, so this adds ADDRESS to the container's +# own dncore_network interface. Every container on that network, node included, then reaches +# it directly. Docker doesn't know about ADDRESS, so it is never taken from anything that +# already answers for it. +# +# (no argument) claim once; fail if ADDRESS can't be claimed +# --loop claim again every GVPN_FIXED_ADDRESS_INTERVAL seconds if ADDRESS is gone, +# e.g. after a network reconnect +set -euo pipefail + +# Must match GNOSISVPN_SERVER_IP in node's entrypoint.sh. +ADDRESS=172.33.255.254 +INTERVAL="${GVPN_FIXED_ADDRESS_INTERVAL:-30}" + +usage() { + echo "usage: ${0} [--loop]" >&2 + exit 64 +} + +log() { echo "gvpn-fixed-address: $*"; } + +claimed() { [ -n "$(ip -4 -o addr show to "${ADDRESS}/32")" ]; } + +claim() { + local route dev dad on_link=' dev ([^ ]+)' + claimed && return 0 + + # The interface on ADDRESS's subnet: "172.33.255.254 dev eth0 src 172.33.0.4 uid 0". + if ! route="$(ip -4 route get "${ADDRESS}" 2>&1)" || [[ "${route}" == *" via "* ]] || + [[ ! "${route}" =~ ${on_link} ]]; then + log "no attached network holds ${ADDRESS} (${route})" >&2 + return 1 + fi + dev="${BASH_REMATCH[1]}" + + if ! dad="$(arping -D -c 2 -w 3 -I "${dev}" "${ADDRESS}" 2>&1)"; then + log "${ADDRESS} is already in use on ${dev}, not claiming it: ${dad//$'\n'/ }" >&2 + return 1 + fi + if ! ip addr add "${ADDRESS}/32" dev "${dev}"; then + log "could not add ${ADDRESS} to ${dev}" >&2 + return 1 + fi + # Neighbours may still have a previous gnosisvpn-server container's MAC for ADDRESS. + arping -U -q -c 2 -I "${dev}" -s "${ADDRESS}" "${ADDRESS}" || true + log "claimed ${ADDRESS} on ${dev}" +} + +case "${1:-}" in +"") claim ;; +--loop) + while sleep "${INTERVAL}"; do + claim || true + done + ;; +*) usage ;; +esac diff --git a/gnosisvpn-server/gvpn-peer-shaping.sh b/gnosisvpn-server/gvpn-peer-shaping.sh new file mode 100755 index 0000000..48739b2 --- /dev/null +++ b/gnosisvpn-server/gvpn-peer-shaping.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Per-peer traffic shaping for the exit node (`gvpn-peer-shaping.sh up|down`). +# Invoked by wg-hooks.sh from the wggvpn PostUp/PreDown hooks. Ported from +# gvpn-exit-os (itself from gnosis_vpn-infrastructure, roles/gnosisvpn-server): +# the Ansible variables arrive here as the environment entrypoint.sh exports. +# +# Shapes (queues) instead of policing (dropping): each peer gets an HTB class +# capped at the per-peer ceil with an fq_codel leaf, so TCP settles smoothly at +# the cap instead of sawtoothing on hard packet loss. The parent class enforces +# the node-wide aggregate cap. +# +# Directions: +# - download (server -> peers): tree on wggvpn egress, classified by dst IP +# - upload (peers -> internet): wggvpn ingress mirrored to an ifb device, tree +# on the ifb classified by src IP. Shaping has to happen on the wggvpn side +# because the uplink only ever sees post-MASQUERADE addresses. +# +# Peers are assigned GVPN_PEER_MIN_OCTET..GVPN_PEER_MAX_OCTET dynamically by +# gnosis_vpn-server, so all classes in that range are provisioned up front and +# packets are steered by a u32 hash table on the last octet (O(1) per packet). +# .1 is the server itself and .255 the broadcast address; both stay unshaped in +# the default class. The range must stay within one /24 (this script only +# varies the last octet), which entrypoint.sh enforces when it derives it. +set -euo pipefail + +WG_IF="wggvpn" +IFB_IF="ifb-gvpn" +# Derived by entrypoint.sh from WIREGUARD_CIDR and MAX_SLOTS, the same way it +# derives config.toml's pool and the rendered WireGuard address. Only `up` +# needs them, so `down` still works when they are absent. +SUBNET_PREFIX="${GVPN_PEER_SUBNET_PREFIX:-}" +PEER_MIN="${GVPN_PEER_MIN_OCTET:-}" +PEER_MAX="${GVPN_PEER_MAX_OCTET:-}" +PEER_RATE_KBIT="${GVPN_PEER_RATE_KBIT:-10000}" +AGGREGATE_KBIT="${GVPN_PEER_SHAPING_AGGREGATE_KBIT:-25000}" +# Guaranteed per-peer share; 253 x 56kbit stays below the aggregate so HTB +# guarantees remain honest. The ceil is what actually caps a peer. +PEER_GUARANTEE_KBIT=56 +# fq_codel tuned to signal congestion by queueing delay rather than drops: a +# dropped packet triggers an end-to-end retransmit across the mixnet, while a +# delayed packet lets the mixnet and TCP adapt to the slower pace. target +# tolerates ~250ms of standing queue (on top of the ~250ms mixnet peer RTT) +# before codel starts dropping, and ECN-capable flows are marked instead of +# dropped. Small quantum for inter-flow latency at low rates. +FQ_CODEL_OPTS="target 250ms interval 1000ms quantum 300 ecn" +# HTB derives a class's quantum from its rate and the parent's r2q, and warns on +# every class when the rates span too wide a range for one r2q (a 56kbit +# guarantee next to a 25Mbit aggregate does). Pin it at one MTU instead, so the +# container log isn't full of tc warnings on every interface sync. +HTB_QUANTUM=1500 + +usage() { + echo "usage: ${0} up|down" >&2 + exit 64 +} + +# Map a peer's last octet to the hex minor id of its class (1:). +peer_classid() { + printf '%x' "${1}" +} + +# IPv4 header offset of the address to classify on: dst for download (traffic +# leaving wggvpn toward peers), src for upload (peer traffic entering wggvpn, +# seen on the ifb mirror). +match_offset() { + case "${1}" in + dst) echo 16 ;; + src) echo 12 ;; + *) + echo "unknown match direction: ${1}" >&2 + return 1 + ;; + esac +} + +# Print the full tc command batch for one device. ${2} is the match direction +# (dst|src). Consumed via `tc -batch -` so the whole tree is built in one +# netlink session instead of ~800 tc invocations. +emit_tree() { + local dev="${1}" dir="${2}" offset octet minor + offset="$(match_offset "${dir}")" + + echo "qdisc add dev ${dev} root handle 1: htb default fff" + echo "class add dev ${dev} parent 1: classid 1:1 htb rate ${AGGREGATE_KBIT}kbit ceil ${AGGREGATE_KBIT}kbit quantum ${HTB_QUANTUM}" + # non-peer traffic (server address, broadcasts) shares the aggregate uncapped + echo "class add dev ${dev} parent 1:1 classid 1:fff htb rate 1000kbit ceil ${AGGREGATE_KBIT}kbit quantum ${HTB_QUANTUM}" + echo "qdisc add dev ${dev} parent 1:fff fq_codel ${FQ_CODEL_OPTS}" + echo "filter add dev ${dev} parent 1: prio 1 handle 2: protocol ip u32 divisor 256" + echo "filter add dev ${dev} parent 1: prio 1 protocol ip u32 ht 800:: match ip ${dir} ${SUBNET_PREFIX}.0/24 hashkey mask 0x000000ff at ${offset} link 2:" + + for octet in $(seq "${PEER_MIN}" "${PEER_MAX}"); do + minor="$(peer_classid "${octet}")" + echo "class add dev ${dev} parent 1:1 classid 1:${minor} htb rate ${PEER_GUARANTEE_KBIT}kbit ceil ${PEER_RATE_KBIT}kbit quantum ${HTB_QUANTUM}" + echo "qdisc add dev ${dev} parent 1:${minor} fq_codel ${FQ_CODEL_OPTS}" + echo "filter add dev ${dev} parent 1: prio 1 protocol ip u32 ht 2:${minor}: match ip ${dir} ${SUBNET_PREFIX}.${octet}/32 flowid 1:${minor}" + done +} + +# tc and the ifb/sch_* modules come from the DAppNode host's kernel, which this +# container cannot load modules into. A host without them gets an unshaped exit +# node and this error in the log, not a dead package. +up() { + # download: replace whatever root qdisc is present (idempotent re-runs) + tc qdisc del dev "${WG_IF}" root 2>/dev/null || true + emit_tree "${WG_IF}" dst | tc -batch - + + # upload: mirror wggvpn ingress onto the ifb and shape there + ip link add "${IFB_IF}" type ifb 2>/dev/null || true + ip link set "${IFB_IF}" up + tc qdisc del dev "${IFB_IF}" root 2>/dev/null || true + emit_tree "${IFB_IF}" src | tc -batch - + tc qdisc del dev "${WG_IF}" ingress 2>/dev/null || true + tc qdisc add dev "${WG_IF}" handle ffff: ingress + tc filter add dev "${WG_IF}" parent ffff: prio 1 protocol ip u32 match u32 0 0 action mirred egress redirect dev "${IFB_IF}" +} + +down() { + tc qdisc del dev "${WG_IF}" ingress 2>/dev/null || true + tc qdisc del dev "${WG_IF}" root 2>/dev/null || true + ip link del "${IFB_IF}" 2>/dev/null || true +} + +require_params() { + local missing=() + [[ -n "${SUBNET_PREFIX}" ]] || missing+=("GVPN_PEER_SUBNET_PREFIX") + [[ -n "${PEER_MIN}" ]] || missing+=("GVPN_PEER_MIN_OCTET") + [[ -n "${PEER_MAX}" ]] || missing+=("GVPN_PEER_MAX_OCTET") + if [[ "${#missing[@]}" -ne 0 ]]; then + echo "not set: ${missing[*]} (derived by entrypoint.sh)" >&2 + return 1 + fi + local value + for value in "${PEER_RATE_KBIT}" "${AGGREGATE_KBIT}"; do + if ! [[ "${value}" =~ ^[1-9][0-9]*$ ]]; then + echo "shaping rates must be positive integers in kbit/s, got '${value}'" >&2 + return 1 + fi + done +} + +main() { + case "${1:-}" in + up) + require_params || exit 64 + up + ;; + down) down ;; + *) usage ;; + esac +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/gnosisvpn-server/wg-hooks.sh b/gnosisvpn-server/wg-hooks.sh new file mode 100755 index 0000000..f8ed8c9 --- /dev/null +++ b/gnosisvpn-server/wg-hooks.sh @@ -0,0 +1,166 @@ +#!/bin/bash +# wg-quick PostUp/PreDown hook for wggvpn (`wg-hooks.sh up|down`). +# +# Sets the exit node's forwarding policy in this container's network namespace, +# where gnosis_vpn-server creates the WireGuard interface. VPN clients are +# untrusted, so they may only reach public destinations: not the DAppNode host, +# not other packages on dncore_network, not the LAN behind the host. +# +# Rule order is the guarantee: the private-range jump is first in FORWARD, the +# client ACCEPT is last, and the policy is DROP. Ported from wg-hooks.sh + +# iptables.rules in the gvpn-exit-os appliance image, which a DAppNode package +# can't reuse as-is because it never sees the host's network namespace (see +# README). +set -euo pipefail + +WG_IF="wggvpn" +PRIVATE_CHAIN="GVPN_PRIVATE" +# Filled by gvpn-blocklist-update.sh, which entrypoint.sh starts; this hook only +# drops traffic against it, and leaves the set in the kernel on `down` so the +# refresher keeps working across interface restarts. +BLOCKLIST_SET="${GVPN_BLOCKLIST_IPSET_NAME:-gvpn-blocklist}" +BLOCKLIST_MAXELEM="${GVPN_BLOCKLIST_MAXELEM:-131072}" +SHAPING_BIN="${GVPN_SHAPING_BIN:-/app/gvpn-peer-shaping.sh}" + +usage() { + echo "usage: ${0} up|down [uplink]" >&2 + exit 64 +} + +[[ $# -ge 3 ]] || usage +action="$1" +client_cidr="$2" +server_address="$3" +# The container's DAppNode-managed interface; the host NATs it the rest of the +# way out. +uplink="${4:-eth0}" + +# Destinations a VPN client must never reach: RFC1918 (the LAN and Docker's +# default network pool), CGNAT, link-local, loopback, "this network" and +# multicast/reserved. +PRIVATE_NETS=( + 10.0.0.0/8 + 172.16.0.0/12 + 192.168.0.0/16 + 100.64.0.0/10 + 169.254.0.0/16 + 127.0.0.0/8 + 0.0.0.0/8 + 224.0.0.0/3 + # DAppNode's dncore_network (every package and the dappmanager). It is not in + # RFC1918, so without this entry the dappmanager stays reachable. + 172.33.0.0/16 +) + +PRIVATE_JUMP=(FORWARD -i "${WG_IF}" -j "${PRIVATE_CHAIN}") +BLOCKLIST_DST_DROP=(FORWARD -i "${WG_IF}" -m set --match-set "${BLOCKLIST_SET}" dst -j DROP) +BLOCKLIST_SRC_DROP=(FORWARD -o "${WG_IF}" -m set --match-set "${BLOCKLIST_SET}" src -j DROP) +CLIENT_ACCEPT=(FORWARD -i "${WG_IF}" -o "${uplink}" -j ACCEPT) +# shellcheck disable=SC2054 # the comma belongs to --ctstate, not the array +RETURN_ACCEPT=(FORWARD -o "${WG_IF}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT) +# The GnosisVPN client pings the WireGuard server address (`connection.ping`, +# default 10.128.0.1) to verify the tunnel. Accept only that echo request; +# everything else from clients is dropped, the control API on port 8000 included. +PING_ACCEPT=(INPUT -s "${client_cidr}" -d "${server_address}/32" -i "${WG_IF}" -p icmp -m icmp --icmp-type 8 -j ACCEPT) +CLIENT_DROP=(INPUT -i "${WG_IF}" -j DROP) +NAT_RULE=(POSTROUTING -s "${client_cidr}" -o "${uplink}" -j MASQUERADE) + +# The subnet this container is attached to, whatever DAppNode assigned, so +# isolation doesn't rely on the static entry above. Read from the kernel's +# route for the interface. +own_network() { + local network + # awk reads all input, so ip never hits a closed pipe and pipefail only + # reports a real ip failure. + if ! network="$(ip -4 route show dev "${uplink}" proto kernel scope link | awk 'NR == 1 { print $1 }')"; then + echo "wg-hooks: cannot read the routes of ${uplink}" >&2 + return 1 + fi + if [[ -z "${network}" ]]; then + echo "wg-hooks: no IPv4 subnet on ${uplink}; only the static ranges are dropped" >&2 + return 0 + fi + printf '%s\n' "${network}" +} + +private_nets() { + local own + own="$(own_network)" || return 1 + printf '%s\n' "${PRIVATE_NETS[@]}" "${own}" | awk 'NF && !seen[$0]++' +} + +up() { + # gnosis_vpn-server can bring the interface up again without PreDown (hard + # kill, interface sync), so clear old rules first to avoid duplicates. + down + + # Rules for a missing interface match nothing and raise no error, which + # would silently leave the control API reachable. Refuse instead. + if ! ip link show dev "${WG_IF}" >/dev/null 2>&1; then + echo "wg-hooks: ${WG_IF} does not exist; refusing to install forwarding rules" >&2 + exit 1 + fi + + # Fail closed first: a client packet arriving mid-install is dropped. + iptables -P FORWARD DROP + iptables -N "${PRIVATE_CHAIN}" 2>/dev/null || iptables -F "${PRIVATE_CHAIN}" + # Collected first, not read from a process substitution (bash ignores its + # failure): a range that can't be determined must stop the hook. + local nets net + nets="$(private_nets)" + while IFS= read -r net; do + iptables -A "${PRIVATE_CHAIN}" -d "${net}" -j DROP + done <<<"${nets}" + # The blocklist rules resolve the set name, so it has to exist first. The + # refresher fills this same set in place with ipset swap. + ipset create "${BLOCKLIST_SET}" hash:net family inet maxelem "${BLOCKLIST_MAXELEM}" -exist + + # -I without a rule number inserts at the head, ahead of anything else, so + # every drop lands in front of the client ACCEPT below. + iptables -I "${BLOCKLIST_DST_DROP[@]}" + iptables -I "${BLOCKLIST_SRC_DROP[@]}" + iptables -I "${PRIVATE_JUMP[@]}" + iptables -A "${CLIENT_ACCEPT[@]}" + iptables -A "${RETURN_ACCEPT[@]}" + iptables -I "${CLIENT_DROP[@]}" + iptables -I "${PING_ACCEPT[@]}" + + # tc and the ifb/sch_* modules come from the DAppNode host's kernel, which this + # container cannot load modules into. A host without them gets an unshaped exit + # node and this line in the log, rather than no exit node at all. + if ! "${SHAPING_BIN}" up; then + echo "wg-hooks: per-peer shaping failed; VPN clients are not rate limited" >&2 + fi + + iptables -t nat -A "${NAT_RULE[@]}" +} + +down() { + iptables -t nat -D "${NAT_RULE[@]}" 2>/dev/null || true + "${SHAPING_BIN}" down 2>/dev/null || true + iptables -D "${PING_ACCEPT[@]}" 2>/dev/null || true + iptables -D "${CLIENT_DROP[@]}" 2>/dev/null || true + iptables -D "${RETURN_ACCEPT[@]}" 2>/dev/null || true + iptables -D "${CLIENT_ACCEPT[@]}" 2>/dev/null || true + iptables -D "${PRIVATE_JUMP[@]}" 2>/dev/null || true + iptables -D "${BLOCKLIST_SRC_DROP[@]}" 2>/dev/null || true + iptables -D "${BLOCKLIST_DST_DROP[@]}" 2>/dev/null || true + iptables -F "${PRIVATE_CHAIN}" 2>/dev/null || true + iptables -X "${PRIVATE_CHAIN}" 2>/dev/null || true + # FORWARD stays DROP on purpose: nothing forwards without the interface, and + # a hook that failed half way must not leave the namespace open. + # The blocklist set stays too: emptying it on every interface sync would + # leave the blocklist inactive until the next refresh. +} + +main() { + case "${action}" in + up) up ;; + down) down ;; + *) usage ;; + esac +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/gnosisvpn-server/wggvpn.conf.tpl b/gnosisvpn-server/wggvpn.conf.tpl index 07e5979..0232e80 100644 --- a/gnosisvpn-server/wggvpn.conf.tpl +++ b/gnosisvpn-server/wggvpn.conf.tpl @@ -1,17 +1,16 @@ -# Rendered by entrypoint.sh from this template. PostUp/PreDown do the NAT a WireGuard exit -# node needs, scoped down to what's possible for a regular (non-core) DAppNode package: no -# network_mode: host, so this container's own network namespace (on dncore_network, like any -# other DAppNode package service - see docker-compose.yml) never reaches the DAppNode host's. -# MASQUERADE targets "eth0" (this container's own interface on its DAppNode-managed network) -# rather than a real public NIC; Docker's own host-level NAT then carries that traffic the -# rest of the way out, same as it does for any other container's egress traffic. Explicit -# DOCKER-USER firewall exceptions aren't included for the same reason (no host namespace -# access) - see README's "Known limitations vs. production" section for why that's not -# actually a gap here. +# Rendered by entrypoint.sh from this template. PostUp/PreDown run wg-hooks.sh, +# which installs the exit node's forwarding policy in this container's own +# network namespace. A regular DAppNode package gets no network_mode: host, so +# it never reaches the host's namespace (see docker-compose.yml). +# VPN clients may only reach public destinations. MASQUERADE targets "eth0" +# (this container's DAppNode-managed interface), and Docker's host-level NAT +# carries the traffic the rest of the way out. There are no DOCKER-USER +# exceptions for the same reason - see README's "Known limitations vs. +# production" for why that's not a gap here. [Interface] Address = __WG_ADDRESS__/32 ListenPort = 51820 PrivateKey = MTU = __WG_MTU__ -PostUp = iptables -t nat -A POSTROUTING -s __WG_CIDR__ -o eth0 -j MASQUERADE -PreDown = iptables -t nat -D POSTROUTING -s __WG_CIDR__ -o eth0 -j MASQUERADE +PostUp = /app/wg-hooks.sh up __WG_CIDR__ __WG_ADDRESS__ +PreDown = /app/wg-hooks.sh down __WG_CIDR__ __WG_ADDRESS__ diff --git a/hoprd.cfg.yaml.tpl b/hoprd.cfg.yaml.tpl index 72b22d0..cb120b7 100644 --- a/hoprd.cfg.yaml.tpl +++ b/hoprd.cfg.yaml.tpl @@ -1,15 +1,32 @@ --- blokli_url: https://blokli-piz-palu.dev.hoprnet.link +hopr: + network: + # Must fit twice in hoprd's 60s loopback window; the 1m default doesn't, and hoprd exits. + probe_interval: 30s + pix: + num_ssa_parts: 4096 + ssa_part_size: 32 + additional_shares: 79 + max_ssas_per_request: 3 + incoming_session_pix: + enforce_pix: false + max_ssa_delivery_time: 20s + max_deposit_wait: 3m + max_recovery_time: 2h + ssas_per_request: 1 session_ip_forwarding: target_allow_list: - # session_ip_forwarding requires a literal ip:port (hostnames are rejected by hoprd's own - # config parser). This package can't pin gnosisvpn-server a static IP on dncore_network - # (not allowed for non-core DAppNode packages - see README), so entrypoint.sh resolves - # gnosisvpn-server's DAppNode DNS alias to its current IP at boot and renders this - # template (see GNOSISVPN_SERVER_HOST in docker-compose.yml) - this file is not used - # directly, only the rendered output at HOPRD_CONFIGURATION_FILE_PATH. + # hoprd needs a literal ip:port. entrypoint.sh replaces the placeholder with + # gnosisvpn-server's fixed address, 172.33.255.254. - "__GNOSISVPN_SERVER_IP__:51820" # gnosisvpn-server's WireGuard interface - "__GNOSISVPN_SERVER_IP__:8000" # gnosisvpn-server's control endpoint + # First match wins: only WireGuard sessions pay. + session_admission_rules: + - target: "udp:__GNOSISVPN_SERVER_IP__:51820" + enforce_pix: true + - target: "*" + enforce_pix: false strategy: allow_recursive: false execution_interval: 15s @@ -72,3 +89,23 @@ strategy: selector: default - AutoRedeeming: minimum_redeem_ticket_value: "1 wei wxHOPR" + - Pix: + strategy: + price_per_byte: "9 000 000 000 wei wxHOPR" + max_ssa_allocation: 3 wxHOPR + max_spend_per_window: 6 wxHOPR + spend_window: 1day + deposit_buffer_period: 500ms + withdrawal_buffer_period: 500ms + # PIX state is sealed with the identity's password and kept on the node-data + # volume. Losing either loses the funds not yet swept. + pix_recovery_password_env: HOPRD_PASSWORD + pix_recovery_db_path: /app/hoprd/data/pix-recovery.redb + pool: + # The pool doesn't read HOPRD_BLOKLI_URL; entrypoint.sh fills in the node's URL. + blokli_url: __BLOKLI_URL__ + submission: relayer + relayer_url: https://api.curvy.box + token: 2 + max_deposit_tracking_time: 3m 20s + state_path: /app/hoprd/data/curvy-pix.redb diff --git a/justfile b/justfile index c39e2f6..5cd89b2 100644 --- a/justfile +++ b/justfile @@ -14,6 +14,37 @@ local-down: local-logs: docker compose -f docker-compose.yml -f docker-compose.local.yml --env-file .env.local logs -f -# Re-validate hoprd.cfg.yaml against the real hoprd 4.0.3 config parser +# Re-validate hoprd.cfg.yaml.tpl against the real hoprd config parser validate-config: ./test/validate-hoprd-config.sh + +# Check node's entrypoint and the settings its stored keys depend on +test-entrypoint: + ./test/entrypoint.sh + +# Check that node reaches gnosisvpn-server at its fixed address +test-fixed-address: + ./test/fixed-address.sh + +# Run the same fixed-address checks with the built gnosisvpn-server image +test-fixed-address-image: + docker build -t gnosisvpn-server:fixed-address-test gnosisvpn-server + FIXED_ADDRESS_TEST_IMAGE=gnosisvpn-server:fixed-address-test ./test/fixed-address.sh + +# Check that VPN clients can't reach local resources through the exit node +test-firewall: + ./test/firewall.sh + +# Run the same firewall checks inside the built gnosisvpn-server image +test-firewall-image: + docker build -t gnosisvpn-server:firewall-test gnosisvpn-server + FIREWALL_TEST_IMAGE=gnosisvpn-server:firewall-test ./test/firewall.sh + +# Check the blocklist and per-peer shaping the exit node applies +test-traffic: + ./test/traffic.sh + +# Run the same blocklist and shaping checks inside the built image +test-traffic-image: + docker build -t gnosisvpn-server:traffic-test gnosisvpn-server + TRAFFIC_TEST_IMAGE=gnosisvpn-server:traffic-test ./test/traffic.sh diff --git a/setup-wizard.yml b/setup-wizard.yml index d3588d6..7254399 100644 --- a/setup-wizard.yml +++ b/setup-wizard.yml @@ -51,14 +51,11 @@ fields: description: |- Upload a custom configuration file for the HOPR node, replacing the shipped `hoprd.cfg.yaml.tpl` (jura-prod GnosisVPN exit-node fleet defaults) entirely - including - its `session_ip_forwarding` entries. gnosisvpn-server's address isn't fixed (it can - differ per install and across a gnosisvpn-server recreate) and hoprd can't resolve it - itself (its config parser requires a literal ip:port, not a hostname), so if you - reference gnosisvpn-server here, use the literal placeholder `__GNOSISVPN_SERVER_IP__` - in place of its address (e.g. `__GNOSISVPN_SERVER_IP__:51820`) - the node substitutes - it with gnosisvpn-server's actual current address on every start, the same way it does - for the shipped config. Uploading here takes priority over the auto-generated config on - every start. + its `session_ip_forwarding` entries, which must list gnosisvpn-server at + `172.33.255.254:51820` and `172.33.255.254:8000` (`__GNOSISVPN_SERVER_IP__` works in + place of the address), and its `Pix` strategy's `pool.blokli_url`, which should be + `__BLOKLI_URL__`. Uploading here takes priority over the auto-generated config on every + start. secret: false - id: HOPRD_SAFE_ADDRESS @@ -146,3 +143,64 @@ fields: secret: false pattern: ^([1-9]|[1-9]\d|1\d{2}|2[0-4]\d|25[0-2])$ patternErrorMessage: Must be a number between 1 and 252 + + - id: GVPN_PEER_RATE_KBIT + target: + type: environment + name: GVPN_PEER_RATE_KBIT + service: gnosisvpn-server + title: Per-client bandwidth limit (advanced) + required: false + description: |- + Each connected VPN client is limited to this rate in each direction, in kbit/s. Shaping + queues packets instead of dropping them, so TCP slows down smoothly at the limit. Leave + empty to use the default (`10000`, i.e. 10 Mbit/s). + secret: false + pattern: ^[1-9][0-9]*$ + patternErrorMessage: Must be a positive whole number of kbit/s, e.g. 10000 + + - id: GVPN_PEER_SHAPING_AGGREGATE_KBIT + target: + type: environment + name: GVPN_PEER_SHAPING_AGGREGATE_KBIT + service: gnosisvpn-server + title: Total bandwidth limit for all clients (advanced) + required: false + description: |- + All connected clients together are limited to this rate, in kbit/s. Keep it below your + uplink speed so the exit node cannot saturate your connection. Leave empty to use the + default (`25000`, i.e. 25 Mbit/s). + secret: false + pattern: ^[1-9][0-9]*$ + patternErrorMessage: Must be a positive whole number of kbit/s, e.g. 25000 + + - id: GVPN_BLOCKLIST_FEEDS + target: + type: environment + name: GVPN_BLOCKLIST_FEEDS + service: gnosisvpn-server + title: Blocklist feeds (advanced) + required: false + description: |- + Space-separated URLs of IPv4 threat feeds, each listing one address or CIDR per line. + Traffic from VPN clients to these addresses, and from them back to clients, is dropped. + Leave empty to use the default: FireHOL level1, the same feed GnosisVPN's own exit nodes + use. + secret: false + pattern: ^(https?://[^ ]+)( (https?://[^ ]+))*$ + patternErrorMessage: Must be one or more space-separated http(s) URLs + + - id: GVPN_BLOCKLIST_REFRESH_INTERVAL_S + target: + type: environment + name: GVPN_BLOCKLIST_REFRESH_INTERVAL_S + service: gnosisvpn-server + title: Blocklist refresh interval (advanced) + required: false + description: |- + How often the blocklist feeds are fetched, in seconds, plus up to one hour of jitter. + Leave empty to use the default (`43200`, twice a day). A refresh that fails, or that + parses to an implausible number of entries, keeps the previous list. + secret: false + pattern: ^[1-9][0-9]*$ + patternErrorMessage: Must be a positive whole number of seconds, e.g. 43200 diff --git a/test/README.md b/test/README.md index c482d99..c199e10 100644 --- a/test/README.md +++ b/test/README.md @@ -36,12 +36,86 @@ and its `iptables` rules live inside that container, not on your real machine - local-down` (or `docker compose down`) tears it down along with both containers, nothing to clean up on the host afterward. -`node`'s `entrypoint.sh` resolves `gnosisvpn-server`'s address at boot (hoprd's -`session_ip_forwarding.target_allow_list` requires a literal ip:port, hostnames are rejected) -and renders it into `hoprd.cfg.yaml.tpl` - see that file and `docker-compose.yml`'s -`GNOSISVPN_SERVER_HOST`. Outside a real DAppNode install there's no DAppNode DNS alias to -resolve, so `docker-compose.local.yml` overrides `GNOSISVPN_SERVER_HOST` to plain -`gnosisvpn-server`, which plain `docker compose` resolves via the project's default network. +The stack's network uses `dncore_network`'s subnet, `172.33.0.0/16`, because +`gnosisvpn-server`'s fixed address `172.33.255.254` is in it (see the README's "Fixed +address"). It can't run on a machine that already has a network on that subnet, such as a +DAppNode. + +## Checking node's rendered config + +```sh +just test-entrypoint # or: test/entrypoint.sh +``` + +Checks that [`entrypoint.sh`](../entrypoint.sh) renders `gnosisvpn-server`'s fixed address +and the PIX pool's Blokli URL without any network, ignores a config an older version rendered +at the upload path, uses a real upload and warns when it doesn't forward to the fixed address, +and `exec`s into hoprd. It also checks that the identity and PIX state paths, their volumes and +the `HOPRD_PASSWORD` variable stay the same, since changing any of them loses existing +installs' identity or PIX funds on update. Only Docker is needed. + +## Checking the fixed address + +```sh +just test-fixed-address # or: test/fixed-address.sh +just test-fixed-address-image # or: FIXED_ADDRESS_TEST_IMAGE= test/fixed-address.sh +``` + +Checks [`gvpn-fixed-address.sh`](../gnosisvpn-server/gvpn-fixed-address.sh) on networks laid +out like DAppNode's, with a stand-in `node` without extra capabilities: `node` reaches the +server at `172.33.255.254:8000`, also after the server is recreated at another Docker address +or reconnected to the network, and a second claimant backs off. + +`test-fixed-address-image` runs the built image's own entrypoint, so `node` talks to the real +control API. Like the local stack, neither runs on a machine with a `172.33.0.0/16` network. + +## Checking the firewall + +```sh +just test-firewall # or: test/firewall.sh +just test-firewall-image # or: FIREWALL_TEST_IMAGE= test/firewall.sh +``` + +Checks that a VPN client can't reach local resources, but can still reach public destinations +and ping the tunnel-check address (see the README's "Firewall" section). Both commands: + +1. Grep `wggvpn.conf.tpl`, the `gnosisvpn-server` Dockerfile and `entrypoint.sh` for the + `wg-hooks.sh` wiring and the early FORWARD DROP. +2. Run `test/firewall-check.sh` in a throwaway privileged container. A veth pair named `wggvpn` + stands in for WireGuard, a dummy `eth1` for the uplink, and a separate network namespace for + the client, so the probes really go through the rules. Only Docker is needed. + +`test-firewall` uses `alpine` with the repo's `wg-hooks.sh` mounted in. `test-firewall-image` +uses the built `gnosisvpn-server` image with nothing mounted or installed, which proves the +image ships the hook and its tools (`bash`, `iproute2`, `iptables`). Building that image needs +access to GnosisVPN's registry: `gcloud auth configure-docker europe-west3-docker.pkg.dev`. + +## Checking the blocklist and shaping + +```sh +just test-traffic # or: test/traffic.sh +just test-traffic-image # or: TRAFFIC_TEST_IMAGE= test/traffic.sh +``` + +Checks the two things the exit node does to forwarded traffic besides isolating it (see the +README's "Blocklist" and "Per-peer shaping" sections). Both commands: + +1. Grep the `gnosisvpn-server` Dockerfile, `entrypoint.sh` and `wg-hooks.sh` for the wiring: the + image ships both scripts, the entrypoint loads the cached list and starts the refresher, and + the hook runs the shaping. +2. Run `test/traffic-check.sh` in a throwaway privileged container. On top of the firewall + suite's veth and client namespace there is a third namespace standing in for a public host + that answers pings, so "the client still reaches the internet" is a real reply rather than a + rule counter. Feeds are `file://` URLs, so no network is needed. + +It covers the blocklist parser and the fail-open paths (a feed below the minimum entry count, an +unreachable feed, reloading from the cache), that a blocklisted destination really is dropped +while a public one is not, and that every peer slot has an HTB class in both directions with +packets actually counted in it. + +`test-traffic` uses `alpine` with the repo's scripts mounted in. `test-traffic-image` uses the +built image with nothing mounted or installed, which proves it ships `ipset` and `curl` as well +as the scripts. ## Validating hoprd.cfg.yaml.tpl on its own @@ -50,7 +124,7 @@ just validate-config # or: test/validate-hoprd-config.sh ``` Runs hoprd's own config validator (`hoprd-cfg --validate-args`) against `hoprd.cfg.yaml.tpl` -(rendered with a dummy IP first), without starting anything. Pulls whatever +(with its placeholders filled in first), without starting anything. Pulls whatever `UPSTREAM_VERSION` currently resolves to so this can start failing if an upstream release changes the schema, independently of any change here. diff --git a/test/entrypoint-stub.sh b/test/entrypoint-stub.sh new file mode 100755 index 0000000..ad99a41 --- /dev/null +++ b/test/entrypoint-stub.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# Stands in for hoprd's /bin/docker-entrypoint.sh. The PID shows whether entrypoint.sh execs. +echo "stub: pid=$$ ppid=${PPID}" diff --git a/test/entrypoint.sh b/test/entrypoint.sh new file mode 100755 index 0000000..6e579d9 --- /dev/null +++ b/test/entrypoint.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# Checks node's entrypoint.sh and the settings its stored keys depend on. See test/README.md. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +SERVER_ADDRESS=172.33.255.254 +# A dynamic address an older version rendered. +STALE_ADDRESS=172.33.0.7 +IMAGE=gnosisvpn-entrypoint-test +CONF_VOLUME=gnosisvpn-entrypoint-test-conf +STUB=test/entrypoint-stub.sh +UPLOAD=/app/hoprd/conf/hoprd.custom.cfg.yaml +TEMPLATE_BLOKLI_URL="$(sed -n 's|^blokli_url: ||p' hoprd.cfg.yaml.tpl)" +BUILD_DIR="" + +failures=0 +pass() { echo "ok: $*"; } +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} +expect_contains() { # $1 haystack, $2 needle, $3 what + case "$1" in + *"$2"*) pass "$3" ;; + *) fail "$3 (missing '$2')" ;; + esac +} +expect_not_contains() { # $1 haystack, $2 needle, $3 what + case "$1" in + *"$2"*) fail "$3 (unexpected '$2')" ;; + *) pass "$3" ;; + esac +} +has_line() { # $1 file, $2 whole line as an extended regex, $3 what + if grep -qxE "$2" "$1"; then pass "$3"; else fail "$3 (no line matching '$2' in $1)"; fi +} + +echo "### settings the identity and PIX state depend on" +# Installs keep their identity and PIX funds across updates only while these stay the same. +# Changing one gives them a new identity, or one hoprd can't decrypt. +has_line docker-compose.yml ' *HOPRD_IDENTITY: "?/app/hoprd/conf/hopr\.id"?' \ + "identity stays at /app/hoprd/conf/hopr.id" +has_line docker-compose.yml ' *- "?node-conf:/app/hoprd/conf"?' "on the node-conf volume" +has_line hoprd.cfg.yaml.tpl ' *state_path: /app/hoprd/data/curvy-pix\.redb' \ + "PIX pool state stays at /app/hoprd/data/curvy-pix.redb" +has_line hoprd.cfg.yaml.tpl ' *pix_recovery_db_path: /app/hoprd/data/pix-recovery\.redb' \ + "PIX recovered keys stay at /app/hoprd/data/pix-recovery.redb" +has_line docker-compose.yml ' *- "?node-data:/app/hoprd/data"?' "on the node-data volume" +has_line docker-compose.yml ' *HOPRD_PASSWORD:.*' "the password stays in HOPRD_PASSWORD" +has_line hoprd.cfg.yaml.tpl ' *pix_recovery_password_env: HOPRD_PASSWORD' "PIX state is sealed with it too" +has_line setup-wizard.yml ' *name: HOPRD_PASSWORD' "which the setup wizard sets" +echo + +cleanup() { + docker volume rm "${CONF_VOLUME}" >/dev/null 2>&1 || true + [ -z "${BUILD_DIR}" ] || rm -rf "${BUILD_DIR}" +} +trap cleanup EXIT +cleanup +BUILD_DIR="$(mktemp -d)" + +printf 'FROM alpine:3\nRUN apk add --no-cache bash\n' >"${BUILD_DIR}/Dockerfile" +docker build -q -t "${IMAGE}" "${BUILD_DIR}" >/dev/null +docker volume create "${CONF_VOLUME}" >/dev/null + +# As PID 1 with no network. HOPRD_CONFIGURATION_FILE_PATH is the upload path, as in older +# installs, to prove the rendered path doesn't come from it. Arguments are extra docker run +# options. +run_entrypoint() { + docker run --rm --network none \ + -v "${CONF_VOLUME}:/app/hoprd/conf" \ + -v "${PWD}/entrypoint.sh:/bin/entrypoint.sh:ro" \ + -v "${PWD}/hoprd.cfg.yaml.tpl:/app/hoprd.cfg.yaml.tpl:ro" \ + -v "${PWD}/${STUB}:/bin/docker-entrypoint.sh:ro" \ + -e HOPRD_CONFIGURATION_FILE_PATH="${UPLOAD}" \ + "$@" \ + --entrypoint /bin/entrypoint.sh \ + "${IMAGE}" 2>&1 || true +} + +# The template as entrypoint.sh renders it. +render_template() { + sed -e "s|__GNOSISVPN_SERVER_IP__|${SERVER_ADDRESS}|g" \ + -e "s|__BLOKLI_URL__|${TEMPLATE_BLOKLI_URL}|g" hoprd.cfg.yaml.tpl +} + +volume_file() { # $1 = file name inside /app/hoprd/conf, empty when absent + docker run --rm -v "${CONF_VOLUME}:/conf" --entrypoint cat "${IMAGE}" "/conf/$1" 2>/dev/null || true +} + +volume_has() { docker run --rm -v "${CONF_VOLUME}:/conf" --entrypoint test "${IMAGE}" -e "/conf/$1"; } + +write_volume_file() { # $1 = file name inside /app/hoprd/conf, stdin = contents + docker run --rm -i -v "${CONF_VOLUME}:/conf" --entrypoint sh "${IMAGE}" -c "cat >'/conf/$1'" +} + +remove_upload() { docker run --rm -v "${CONF_VOLUME}:/conf" --entrypoint rm "${IMAGE}" -f /conf/hoprd.custom.cfg.yaml; } + +echo "### boot without a network" +out="$(run_entrypoint)" +rendered="$(volume_file hoprd.generated.cfg.yaml)" +expect_contains "${rendered}" "- \"${SERVER_ADDRESS}:51820\"" "renders gnosisvpn-server's WireGuard address" +expect_contains "${rendered}" "- \"${SERVER_ADDRESS}:8000\"" "renders gnosisvpn-server's control address" +expect_contains "${rendered}" " blokli_url: ${TEMPLATE_BLOKLI_URL}" "gives the PIX pool the node's Blokli URL" +expect_not_contains "${rendered}" "__" "leaves no placeholder behind" +expect_contains "${out}" "stub: pid=1 ppid=0" "execs into hoprd instead of forking" +if volume_has hoprd.custom.cfg.yaml; then + fail "wrote to the setup wizard's upload path" +else + pass "leaves the setup wizard's upload path alone" +fi + +echo +echo "### leftover rendered config at the upload path (what older versions wrote there)" +sed "s|__GNOSISVPN_SERVER_IP__|${STALE_ADDRESS}|g" hoprd.cfg.yaml.tpl | write_volume_file hoprd.custom.cfg.yaml +out="$(run_entrypoint)" +rendered="$(volume_file hoprd.generated.cfg.yaml)" +expect_contains "${out}" "ignoring ${UPLOAD}" "ignores it" +expect_contains "${rendered}" "- \"${SERVER_ADDRESS}:51820\"" "renders the fixed address instead" +expect_not_contains "${rendered}" "${STALE_ADDRESS}" "does not let it pin the stale address" +if volume_has hoprd.custom.cfg.yaml; then + pass "leaves the operator's file in place" +else + fail "deleted the file at the upload path" +fi + +echo +echo "### leftover from an older release whose template had other settings" +# Releases before the probe_interval setting rendered their template without it. +sed -e '/^hopr:/,/probe_interval/d' -e "s|__GNOSISVPN_SERVER_IP__|${STALE_ADDRESS}|g" \ + hoprd.cfg.yaml.tpl | write_volume_file hoprd.custom.cfg.yaml +out="$(run_entrypoint)" +rendered="$(volume_file hoprd.generated.cfg.yaml)" +expect_contains "${out}" "ignoring ${UPLOAD}" "ignores it" +if [ "${rendered}" = "$(render_template)" ]; then + pass "renders the current template instead" +else + fail "did not render the current template: $(diff <(echo "${rendered}") <(render_template) | head -n 5)" +fi + +echo +echo "### HOPRD_BLOKLI_URL from the setup wizard" +out="$(run_entrypoint -e HOPRD_BLOKLI_URL=https://wizard.invalid)" +rendered="$(volume_file hoprd.generated.cfg.yaml)" +expect_contains "${rendered}" " blokli_url: https://wizard.invalid" "gives it to the PIX pool" + +echo +echo "### a real upload using the placeholder" +sed 's|^blokli_url: .*|blokli_url: https://uploaded.invalid|' hoprd.cfg.yaml.tpl | + write_volume_file hoprd.custom.cfg.yaml +out="$(run_entrypoint)" +rendered="$(volume_file hoprd.generated.cfg.yaml)" +expect_contains "${out}" "using uploaded ${UPLOAD}" "uses it" +expect_contains "${rendered}" "blokli_url: https://uploaded.invalid" "carries its own content through" +expect_contains "${rendered}" " blokli_url: https://uploaded.invalid" "gives the PIX pool its Blokli URL" +expect_contains "${rendered}" "- \"${SERVER_ADDRESS}:51820\"" "replaces its placeholder with the fixed address" +expect_not_contains "${out}" "WARNING" "does not warn" + +echo +echo "### a real upload that writes the fixed address out" +sed -e 's|^blokli_url: .*|blokli_url: https://uploaded.invalid|' \ + -e "s|__GNOSISVPN_SERVER_IP__|${SERVER_ADDRESS}|g" hoprd.cfg.yaml.tpl | + write_volume_file hoprd.custom.cfg.yaml +out="$(run_entrypoint)" +expect_contains "${out}" "using uploaded ${UPLOAD}" "uses it" +expect_not_contains "${out}" "WARNING" "does not warn" + +echo +echo "### a real upload that hardcodes another address" +# Without the template's comment on the WireGuard entry, which marks older versions' output. +sed -e 's|^blokli_url: .*|blokli_url: https://uploaded.invalid|' \ + -e "s|__GNOSISVPN_SERVER_IP__:51820\".*|${STALE_ADDRESS}:51820\"|" \ + -e "s|__GNOSISVPN_SERVER_IP__|${STALE_ADDRESS}|g" hoprd.cfg.yaml.tpl | + write_volume_file hoprd.custom.cfg.yaml +out="$(run_entrypoint)" +rendered="$(volume_file hoprd.generated.cfg.yaml)" +expect_contains "${out}" "using uploaded ${UPLOAD}" "uses it" +expect_contains "${out}" "WARNING: ${UPLOAD} does not forward to gnosisvpn-server at ${SERVER_ADDRESS}" "warns that hoprd won't reach gnosisvpn-server" +expect_contains "${rendered}" "- \"${STALE_ADDRESS}:51820\"" "keeps what the operator wrote" + +echo +echo "### a real upload without PIX" +sed -e 's|^blokli_url: .*|blokli_url: https://uploaded.invalid|' -e '/^ - Pix:/,$d' hoprd.cfg.yaml.tpl | + write_volume_file hoprd.custom.cfg.yaml +out="$(run_entrypoint)" +expect_contains "${out}" "using uploaded ${UPLOAD}" "uses it" +expect_contains "${out}" "WARNING: ${UPLOAD} does not set the Pix strategy's pool.blokli_url to __BLOKLI_URL__" \ + "warns that PIX payments may not work" + +echo +echo "### a real upload with blokli_url on its own line" +sed 's|^blokli_url: \(.*\)|blokli_url:\n \1|' hoprd.cfg.yaml.tpl | write_volume_file hoprd.custom.cfg.yaml +out="$(run_entrypoint)" +expect_contains "${out}" "no blokli_url for the PIX pool" "stops with a clear error" +expect_not_contains "${out}" "stub: pid" "does not start hoprd" + +echo +echo "### a real upload with a space before blokli_url's colon" +sed 's|^blokli_url: |blokli_url : |' hoprd.cfg.yaml.tpl | write_volume_file hoprd.custom.cfg.yaml +out="$(run_entrypoint)" +expect_contains "${out}" "no blokli_url for the PIX pool" "stops with a clear error" +expect_not_contains "${out}" "stub: pid" "does not start hoprd" +remove_upload + +echo +if [ "${failures}" -ne 0 ]; then + echo "${failures} check(s) failed" >&2 + exit 1 +fi +echo "all checks passed" diff --git a/test/firewall-check.sh b/test/firewall-check.sh new file mode 100755 index 0000000..26fdd24 --- /dev/null +++ b/test/firewall-check.sh @@ -0,0 +1,359 @@ +#!/bin/bash +# Runs inside a throwaway privileged container (see test/firewall.sh), so the +# rules wg-hooks.sh installs land in a network namespace the test can discard. +# A veth pair named wggvpn stands in for WireGuard and a second namespace for a +# VPN client, so probes are really forwarded across wggvpn, not just checked +# against the ruleset. +set -euo pipefail + +HOOK=/app/wg-hooks.sh +WG_IF=wggvpn +CHAIN_PRIVATE=GVPN_PRIVATE +# The hook also runs the per-peer shaping; that is test/traffic-check.sh's +# subject, so stub it out here rather than mount it and depend on tc and ifb. +export GVPN_SHAPING_BIN=/bin/true +CLIENT_CIDR=10.128.0.0/24 +SERVER_ADDRESS=10.128.0.1 +CLIENT_ADDRESS=10.128.0.2 +# The uplink is a dummy on a public-looking range, so the subnet wg-hooks.sh +# derives from it can't be confused with a static private range. +UPLINK=eth1 +UPLINK_SUBNET=198.51.100.0/24 +# TEST-NET-3, routed out the uplink: public but never reachable, so a probe to +# it can only prove that FORWARD accepted it. +PUBLIC_PROBE=203.0.113.1 +CLIENT_NS=vpn-client +CLIENT_IF=veth-client + +# Every destination the hook should drop, including the uplink subnet it +# derives at runtime. +DROPPED_NETS=( + 10.0.0.0/8 + 172.16.0.0/12 + 192.168.0.0/16 + 100.64.0.0/10 + 169.254.0.0/16 + 127.0.0.0/8 + 0.0.0.0/8 + 224.0.0.0/3 + 172.33.0.0/16 + "${UPLINK_SUBNET}" +) + +failures=0 +pass() { echo "ok: $*"; } +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +setup() { + # The uplink is the default route, like eth0 on the real exit node, so a + # private destination the hook missed would leave through it and match the + # client ACCEPT. + ip link add "${UPLINK}" type dummy + ip addr add 198.51.100.10/24 dev "${UPLINK}" + ip link set "${UPLINK}" up + ip route replace default dev "${UPLINK}" + + # The tunnel, and the client behind it. + ip netns add "${CLIENT_NS}" + ip link add "${WG_IF}" type veth peer name "${CLIENT_IF}" + ip link set "${CLIENT_IF}" netns "${CLIENT_NS}" + ip addr add "${SERVER_ADDRESS}/24" dev "${WG_IF}" + ip link set "${WG_IF}" up + ip -n "${CLIENT_NS}" addr add "${CLIENT_ADDRESS}/24" dev "${CLIENT_IF}" + ip -n "${CLIENT_NS}" link set "${CLIENT_IF}" up + ip -n "${CLIENT_NS}" link set lo up + # Full tunnel: all client traffic goes through wggvpn (the client's + # wireguard.allowed_ips defaults to 0.0.0.0/0). + ip -n "${CLIENT_NS}" route add default via "${SERVER_ADDRESS}" + + # An address outside the uplink and tunnel networks, standing in for the + # exit node's other interfaces. + OTHER_ADDRESS="$(ip -4 -o addr show dev eth0 | awk '{ print $4 }' | cut -d/ -f1)" + [ -n "${OTHER_ADDRESS}" ] || { + echo "no IPv4 address on eth0 to probe the control API on" >&2 + exit 1 + } +} + +# Packet count of the rule in CHAIN whose `iptables -S -v` line contains SPEC, +# with the `-c PACKETS BYTES` counters stripped. Counters are zeroed before +# every probe, so the count belongs to that probe. +counter() { + local chain="$1" spec="$2" + iptables -t filter -S "${chain}" -v | awk -v want="${spec}" -v prefix="-A ${chain} " ' + { + line = $0 + pkts = "missing" + if (match(line, / -c [0-9]+ [0-9]+/)) { + split(substr(line, RSTART + 4, RLENGTH - 4), counts, " ") + pkts = counts[1] + line = substr(line, 1, RSTART - 1) substr(line, RSTART + RLENGTH) + } + # This chain only, so a spec cannot match a rule somewhere else. + if (index(line, prefix) == 1 && index(line, want) > 0) { print pkts; found = 1; exit } + } + END { if (!found) print "missing" } + ' +} + +# expect_rule_count CHAIN SPEC COUNT DESCRIPTION +expect_rule_count() { + local chain="$1" spec="$2" count="$3" description="$4" actual + actual="$(iptables -t filter -S "${chain}" | grep -cF -- "${spec}" || true)" + if [ "${actual}" -eq "${count}" ]; then + pass "${description}" + else + fail "${description}: ${actual} rules match '${spec}' in ${chain}, expected ${count}" + fi +} + +expect_matched() { + local chain="$1" spec="$2" description="$3" pkts + pkts="$(counter "${chain}" "${spec}")" + if [ "${pkts}" = missing ]; then + fail "${description}: no rule '${spec}' in ${chain}" + elif [ "${pkts}" -lt 1 ]; then + fail "${description}: rule '${spec}' in ${chain} matched ${pkts} packets" + else + pass "${description}" + fi +} + +expect_unmatched() { + local chain="$1" spec="$2" description="$3" pkts + pkts="$(counter "${chain}" "${spec}")" + if [ "${pkts}" = missing ]; then + fail "${description}: no rule '${spec}' in ${chain}" + elif [ "${pkts}" -ne 0 ]; then + fail "${description}: rule '${spec}' in ${chain} matched ${pkts} packets" + else + pass "${description}" + fi +} + +# expect_no_private_drop DESCRIPTION: none of the private-range rules matched. +expect_no_private_drop() { + local description="$1" net pkts + for net in "${DROPPED_NETS[@]}"; do + pkts="$(counter "${CHAIN_PRIVATE}" "-d ${net} -j DROP")" + if [ "${pkts}" = missing ]; then + fail "${description}: no rule for ${net} in ${CHAIN_PRIVATE}" + return 0 + elif [ "${pkts}" -ne 0 ]; then + fail "${description}: ${CHAIN_PRIVATE} dropped ${pkts} packets to ${net}" + return 0 + fi + done + pass "${description}" +} + +# probe_from_client DESTINATION: one ICMP echo sent as the VPN client. +probe_from_client() { + ip netns exec "${CLIENT_NS}" ping -c 1 -W 1 "$1" >/dev/null 2>&1 || true +} + +zero_counters() { + local chain + for chain in FORWARD INPUT "${CHAIN_PRIVATE}"; do + iptables -t filter -Z "${chain}" + done +} + +setup + +# --- the hook installs isolation, and installing it twice does not stack ------ +"${HOOK}" up "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" +"${HOOK}" up "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + +forward_policy="$(iptables -t filter -S FORWARD | sed -n '1p')" +if [ "${forward_policy}" = "-P FORWARD DROP" ]; then + pass "FORWARD policy is DROP" +else + fail "FORWARD policy is '${forward_policy}', expected '-P FORWARD DROP'" +fi + +# hoprd opens new connections to the control API and WireGuard port from +# dncore_network via eth0, so INPUT must stay ACCEPT or the exit node breaks. +# Only the tunnel is filtered. +input_policy="$(iptables -t filter -S INPUT | sed -n '1p')" +if [ "${input_policy}" = "-P INPUT ACCEPT" ]; then + pass "INPUT policy is ACCEPT, so hoprd can still reach the exit node" +else + fail "INPUT policy is '${input_policy}', expected '-P INPUT ACCEPT'" +fi + +# One of each, not a stack of them from the second `up`. +expect_rule_count FORWARD "-i ${WG_IF}" 3 "FORWARD has one private jump, one blocklist drop and one client accept" +expect_rule_count INPUT "-i ${WG_IF}" 2 "INPUT has exactly one ping accept and one client drop" + +first_forward_rule="$(iptables -t filter -S FORWARD | sed -n '2p')" +if [ "${first_forward_rule}" = "-A FORWARD -i ${WG_IF} -j ${CHAIN_PRIVATE}" ]; then + pass "the private-range jump is the first FORWARD rule" +else + fail "first FORWARD rule is '${first_forward_rule}', expected the ${CHAIN_PRIVATE} jump" +fi + +for net in "${DROPPED_NETS[@]}"; do + if iptables -t filter -S "${CHAIN_PRIVATE}" | grep -qxF -- "-A ${CHAIN_PRIVATE} -d ${net} -j DROP"; then + pass "${CHAIN_PRIVATE} drops ${net}" + else + fail "${CHAIN_PRIVATE} does not drop ${net}" + fi +done + +masquerade_count="$(iptables -t nat -S POSTROUTING | grep -cF -- "-s ${CLIENT_CIDR} -o ${UPLINK} -j MASQUERADE" || true)" +if [ "${masquerade_count}" -eq 1 ]; then + pass "exactly one MASQUERADE rule for ${CLIENT_CIDR}" +else + fail "found ${masquerade_count} MASQUERADE rules for ${CLIENT_CIDR}, expected 1" +fi + +# --- a VPN client cannot reach private destinations --------------------------- +zero_counters +probe_from_client 192.168.1.1 +expect_matched "${CHAIN_PRIVATE}" "-d 192.168.0.0/16" "client to a LAN address hits the private drop" +expect_unmatched FORWARD "-i ${WG_IF} -o ${UPLINK} -j ACCEPT" "client to a LAN address is not forwarded" + +zero_counters +probe_from_client 172.33.0.1 +expect_matched "${CHAIN_PRIVATE}" "-d 172.33.0.0/16" "client to the DAppNode host hits the private drop" +expect_unmatched FORWARD "-i ${WG_IF} -o ${UPLINK} -j ACCEPT" "client to the DAppNode host is not forwarded" + +zero_counters +probe_from_client 198.51.100.20 +expect_matched "${CHAIN_PRIVATE}" "-d ${UPLINK_SUBNET}" "client to another host on the exit's own network hits the private drop" + +zero_counters +probe_from_client 10.99.0.1 +expect_matched "${CHAIN_PRIVATE}" "-d 10.0.0.0/8" "client to an address in the tunnel's own /8 hits the private drop" + +# --- a VPN client still reaches public destinations --------------------------- +zero_counters +probe_from_client "${PUBLIC_PROBE}" +expect_matched FORWARD "-i ${WG_IF} -o ${UPLINK} -j ACCEPT" "client to a public address is forwarded" +expect_no_private_drop "client to a public address is not dropped by the private ranges" + +# --- the client's tunnel-verification ping still reaches the exit ------------- +zero_counters +probe_from_client "${SERVER_ADDRESS}" +expect_matched INPUT "-i ${WG_IF} -p icmp -m icmp --icmp-type 8 -j ACCEPT" "client ping to the WireGuard address is accepted" +expect_unmatched INPUT "-i ${WG_IF} -j DROP" "client ping to the WireGuard address is not dropped" + +# --- ...and nothing else from the tunnel reaches the exit --------------------- +# The drop is protocol- and port-agnostic, so an echo to the exit's own +# address proves the control API on it is unreachable. +zero_counters +probe_from_client "${OTHER_ADDRESS}" +expect_matched INPUT "-i ${WG_IF} -j DROP" "client to the exit's other addresses is dropped" + +# --- down removes everything the hook installed ------------------------------- +"${HOOK}" down "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + +if iptables -t filter -S "${CHAIN_PRIVATE}" >/dev/null 2>&1; then + fail "${CHAIN_PRIVATE} still exists after down" +else + pass "${CHAIN_PRIVATE} is gone after down" +fi +if iptables -t filter -S FORWARD | grep -qF -- "-i ${WG_IF}"; then + fail "FORWARD still has ${WG_IF} rules after down" +else + pass "FORWARD has no ${WG_IF} rules after down" +fi +forward_policy_after_down="$(iptables -t filter -S FORWARD | sed -n '1p')" +if [ "${forward_policy_after_down}" = "-P FORWARD DROP" ]; then + pass "FORWARD stays DROP after down" +else + fail "FORWARD policy is '${forward_policy_after_down}' after down, expected DROP" +fi +if iptables -t nat -S POSTROUTING | grep -qF -- "-j MASQUERADE"; then + fail "POSTROUTING still masquerades after down" +else + pass "POSTROUTING has no MASQUERADE rule after down" +fi + +# --- a hook that fails half way leaves the namespace closed ------------------- +# Fail each install step in turn with a mocked iptables on PATH, then check the +# half-installed state still isolates clients: policy DROP, and the private +# jump before any ACCEPT. +closed_after_partial_install() { + local policy first + policy="$(iptables -t filter -S FORWARD | sed -n '1p')" + [ "${policy}" = "-P FORWARD DROP" ] || return 1 + first="$(iptables -t filter -S FORWARD | sed -n '2p')" + if iptables -t filter -S FORWARD | grep -q -- "-j ACCEPT"; then + [ "${first}" = "-A FORWARD -i ${WG_IF} -j ${CHAIN_PRIVATE}" ] || return 1 + fi + if iptables -t filter -S INPUT | grep -q -- "-i ${WG_IF} .*-j ACCEPT"; then + iptables -t filter -S INPUT | grep -qxF -- "-A INPUT -i ${WG_IF} -j DROP" || return 1 + fi + return 0 +} + +mock_bin="$(mktemp -d)" +real_iptables="$(command -v iptables)" +cat >"${mock_bin}/iptables" <&2 + exit 1 +fi +exec "${real_iptables}" "\$@" +MOCK +chmod +x "${mock_bin}/iptables" + +for step in "-j ACCEPT" "--ctstate" "--icmp-type"; do + "${HOOK}" down "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + if FAIL_ON="${step}" PATH="${mock_bin}:${PATH}" \ + "${HOOK}" up "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" 2>/dev/null; then + fail "the hook reported success although installing '${step}' failed" + else + pass "the hook fails when installing '${step}' fails" + fi + if closed_after_partial_install; then + pass "a half-installed hook (failed '${step}') still isolates VPN clients" + else + fail "after failing '${step}' a VPN client is no longer isolated" + fi +done +"${HOOK}" down "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + +# --- a hook that cannot read the uplink's subnet refuses to install ---------- +# Without the uplink subnet only the static ranges cover the exit's own +# network, so an `ip` failure must stop the hook. +real_ip="$(command -v ip)" +cat >"${mock_bin}/ip" <&2 + exit 1 +fi +exec "${real_ip}" "\$@" +MOCK +chmod +x "${mock_bin}/ip" + +if PATH="${mock_bin}:${PATH}" \ + "${HOOK}" up "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" 2>/dev/null; then + fail "the hook reported success although reading the routes of ${UPLINK} failed" +else + pass "the hook fails when reading the routes of ${UPLINK} fails" +fi +if closed_after_partial_install; then + pass "a hook that could not read the routes of ${UPLINK} still isolates VPN clients" +else + fail "after failing to read the routes of ${UPLINK} a VPN client is no longer isolated" +fi +"${HOOK}" down "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" +rm -rf "${mock_bin}" + +if [ "${failures}" -ne 0 ]; then + echo "--- final ruleset ---" >&2 + iptables -t filter -S >&2 + iptables -t nat -S >&2 + echo "${failures} check(s) failed" >&2 + exit 1 +fi +echo "all firewall checks passed" diff --git a/test/firewall.sh b/test/firewall.sh new file mode 100755 index 0000000..9baa557 --- /dev/null +++ b/test/firewall.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Checks that a VPN client tunnelled through this exit node can't reach private +# addresses: first that the image runs the hook, then how the rules behave in a +# throwaway privileged container (see test/firewall-check.sh). +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +# wg-quick runs PostUp/PreDown from the rendered config, so a template that +# stops calling the hook ships an unfiltered exit node. Without +# --sync-wg-interface gnosis_vpn-server never runs wg-quick at all. +grep -q '^PostUp = /app/wg-hooks.sh up ' gnosisvpn-server/wggvpn.conf.tpl +grep -q '^PreDown = /app/wg-hooks.sh down ' gnosisvpn-server/wggvpn.conf.tpl +grep -q '^COPY .*wg-hooks.sh' gnosisvpn-server/Dockerfile +grep -q -- '--sync-wg-interface' gnosisvpn-server/entrypoint.sh +echo "ok: wggvpn.conf.tpl runs the hook, the image ships it, and the server syncs the interface" +# wg-quick brings wggvpn up before PostUp runs the hook, so FORWARD must be +# DROP before the server starts. +grep -q '^iptables -P FORWARD DROP$' gnosisvpn-server/entrypoint.sh +echo "ok: entrypoint.sh closes FORWARD before the server brings wggvpn up" + +if [ -n "${FIREWALL_TEST_IMAGE:-}" ]; then + # Tests the image's own /app/wg-hooks.sh, bash, iproute2 and iptables, so + # nothing is mounted or installed. --entrypoint skips entrypoint.sh, which + # would start the server. + docker run --rm --privileged --entrypoint bash \ + -v "${PWD}/test/firewall-check.sh:/firewall-check.sh:ro" \ + "${FIREWALL_TEST_IMAGE}" \ + /firewall-check.sh +else + docker run --rm --privileged \ + -v "${PWD}/gnosisvpn-server/wg-hooks.sh:/app/wg-hooks.sh:ro" \ + -v "${PWD}/test/firewall-check.sh:/firewall-check.sh:ro" \ + alpine:3 \ + sh -c 'apk add --no-cache --quiet bash iptables iproute2 ipset >/dev/null && bash /firewall-check.sh' +fi diff --git a/test/fixed-address.sh b/test/fixed-address.sh new file mode 100755 index 0000000..23fa3ae --- /dev/null +++ b/test/fixed-address.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# Checks that node reaches gnosisvpn-server at its fixed address. See test/README.md. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +ADDRESS=172.33.255.254 +PREFIX=gnosisvpn-fixed-address-test +DNCORE="${PREFIX}-dncore" +DNPRIVATE="${PREFIX}-dnprivate" +SERVER="${PREFIX}-server" +NODE="${PREFIX}-node" +FILLER="${PREFIX}-filler" +IMAGE="${PREFIX}" +# Seconds between the server's re-claims. +INTERVAL=2 +BUILD_DIR="" + +wired() { # grep arguments; stops the test naming the missing line + grep -q "$@" || { + echo "FAIL: no match for grep $*" >&2 + exit 1 + } +} +wired -x "ADDRESS=${ADDRESS}" gnosisvpn-server/gvpn-fixed-address.sh +wired -x "GNOSISVPN_SERVER_IP=${ADDRESS}" entrypoint.sh +wired '^COPY .*gvpn-fixed-address.sh' gnosisvpn-server/Dockerfile +wired '^RUN apk add .*iputils-arping' gnosisvpn-server/Dockerfile +wired -x '/app/gvpn-fixed-address.sh' gnosisvpn-server/entrypoint.sh +wired -x '/app/gvpn-fixed-address.sh --loop &' gnosisvpn-server/entrypoint.sh +echo "ok: node and gnosisvpn-server agree on ${ADDRESS}, and the image claims and keeps it" + +failures=0 +pass() { echo "ok: $*"; } +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +cleanup() { + docker rm -f "${SERVER}" "${NODE}" "${FILLER}" >/dev/null 2>&1 || true + docker network rm "${DNCORE}" "${DNPRIVATE}" >/dev/null 2>&1 || true + [ -z "${BUILD_DIR}" ] || rm -rf "${BUILD_DIR}" +} +trap cleanup EXIT +cleanup +BUILD_DIR="$(mktemp -d)" + +# Stand-ins for node, the second claimant and the server. The server answers on 8000 with its +# hostname. +cat >"${BUILD_DIR}/Dockerfile" <<'EOF' +FROM alpine:3 +RUN apk add --no-cache bash iproute2 iputils-arping && \ + printf '#!/bin/sh\nread -r _\nhostname\n' >/answer && chmod +x /answer +EOF +docker build -q -t "${IMAGE}" "${BUILD_DIR}" >/dev/null + +# dncore_network and dnprivate_network, which DAppNode attaches every service to. +docker network create --subnet 172.33.0.0/16 "${DNCORE}" >/dev/null +docker network create --subnet 10.20.0.0/24 "${DNPRIVATE}" >/dev/null + +if [ -n "${FIXED_ADDRESS_TEST_IMAGE:-}" ]; then + # The image's own entrypoint and gnosis_vpn-server's control API on 8000. + SERVER_RUN=(-e WIREGUARD_CIDR=10.128.0.0/24 -e GVPN_BLOCKLIST_FEEDS=file:///nonexistent + "${FIXED_ADDRESS_TEST_IMAGE}") + CLAIMANT_RUN=(--entrypoint /app/gvpn-fixed-address.sh "${FIXED_ADDRESS_TEST_IMAGE}") +else + SERVER_RUN=(-v "${PWD}/gnosisvpn-server/gvpn-fixed-address.sh:/app/gvpn-fixed-address.sh:ro" + "${IMAGE}" bash -c + '/app/gvpn-fixed-address.sh && { /app/gvpn-fixed-address.sh --loop & } && exec nc -lk -p 8000 -e /answer') + CLAIMANT_RUN=(-v "${PWD}/gnosisvpn-server/gvpn-fixed-address.sh:/app/gvpn-fixed-address.sh:ro" + --entrypoint /app/gvpn-fixed-address.sh "${IMAGE}") +fi + +docker_address() { # $1 container: its Docker-assigned address on dncore_network + docker inspect -f "{{(index .NetworkSettings.Networks \"${DNCORE}\").IPAddress}}" "$1" +} + +start_server() { + docker run -d --name "${SERVER}" --cap-add NET_ADMIN \ + --network "${DNCORE}" --network "${DNPRIVATE}" \ + -e GVPN_FIXED_ADDRESS_INTERVAL="${INTERVAL}" "${SERVER_RUN[@]}" >/dev/null +} + +claims() { # $1 = how many claims the server's log must show + [ "$(docker logs "${SERVER}" 2>&1 | grep -c "claimed ${ADDRESS} on ")" -ge "$1" ] +} + +wait_for() { # $1 = seconds, rest = command + local seconds="$1" + shift + for _ in $(seq 1 "${seconds}"); do + "$@" && return 0 + sleep 1 + done + return 1 +} + +answer() { # the first line the server sends back to node on ADDRESS:8000, empty if none + docker exec "${NODE}" bash -c \ + "exec 3<>/dev/tcp/${ADDRESS}/8000 && printf 'GET / HTTP/1.0\r\n\r\n' >&3 && timeout 5 head -n1 <&3" \ + 2>/dev/null || true +} + +answers() { [ -n "$(answer)" ]; } + +node_neighbour() { # node's cached MAC for ADDRESS + docker exec "${NODE}" awk -v a="${ADDRESS}" '$1 == a { print $4 }' /proc/net/arp +} + +server_mac() { docker inspect -f "{{(index .NetworkSettings.Networks \"${DNCORE}\").MacAddress}}" "${SERVER}"; } + +neighbour_is_server() { [ "$(node_neighbour)" = "$(server_mac)" ]; } + +# No extra capabilities, like node. +docker run -d --name "${NODE}" --network "${DNCORE}" --network "${DNPRIVATE}" \ + --entrypoint sleep "${IMAGE}" 3600 >/dev/null + +echo "### first start" +start_server +if wait_for 30 claims 1; then + pass "claims ${ADDRESS}" +else + fail "never claimed ${ADDRESS}: $(docker logs "${SERVER}" 2>&1 | tail -n 5)" +fi +first_address="$(docker_address "${SERVER}")" +dev_addresses="$(docker exec "${SERVER}" sh -c \ + "ip -4 -o addr show dev \$(ip -4 -o addr show to ${ADDRESS}/32 | awk '{ print \$2 }')")" +case "${dev_addresses}" in +*" ${first_address}/16 "*) pass "keeps Docker's own address ${first_address} on the same interface" ;; +*) fail "the interface holding ${ADDRESS} lost Docker's address ${first_address}: ${dev_addresses}" ;; +esac +if wait_for 30 answers; then + pass "node reaches the server at ${ADDRESS}:8000 ($(answer))" +else + fail "node gets no answer from ${ADDRESS}:8000" +fi + +echo +echo "### a second claimant while the address is held" +if out="$(docker run --rm --cap-add NET_ADMIN --network "${DNCORE}" "${CLAIMANT_RUN[@]}" 2>&1)"; then + fail "claimed an address another container holds: ${out}" +else + case "${out}" in + *"${ADDRESS} is already in use"*) pass "backs off and says why" ;; + *) fail "failed for another reason: ${out}" ;; + esac +fi +if answers; then pass "the server keeps answering"; else fail "the server stopped answering"; fi + +echo +echo "### recreated at another Docker address" +docker rm -f "${SERVER}" >/dev/null +# Takes the old server's Docker address, so the new one gets another. +docker run -d --name "${FILLER}" --network "${DNCORE}" --entrypoint sleep "${IMAGE}" 3600 >/dev/null +start_server +wait_for 30 claims 1 || true +second_address="$(docker_address "${SERVER}")" +if [ "${second_address}" != "${first_address}" ]; then + pass "Docker moved the server from ${first_address} to ${second_address}" +else + fail "Docker gave the server ${first_address} again, so this proves nothing" +fi +# node has the old server's MAC cached for ADDRESS. +if wait_for 3 neighbour_is_server; then + pass "node's neighbour entry for ${ADDRESS} follows the new server" +else + fail "node still sends ${ADDRESS} to $(node_neighbour), the server is $(server_mac)" +fi +if wait_for 30 answers; then + pass "node reaches the new server at the same ${ADDRESS}:8000" +else + fail "node gets no answer from the new server at ${ADDRESS}:8000" +fi + +echo +echo "### reconnected to dncore_network without a restart" +# The new interface doesn't have the address. +docker network disconnect "${DNCORE}" "${SERVER}" +docker network connect "${DNCORE}" "${SERVER}" +if wait_for $((INTERVAL * 5)) claims 2; then + pass "claims ${ADDRESS} again" +else + fail "did not claim ${ADDRESS} again" +fi +if wait_for 10 answers; then + pass "node reaches the server at ${ADDRESS}:8000 again" +else + fail "node gets no answer from ${ADDRESS}:8000 after the reconnect" +fi + +echo +if [ "${failures}" -ne 0 ]; then + echo "${failures} check(s) failed" >&2 + exit 1 +fi +echo "all checks passed" diff --git a/test/traffic-check.sh b/test/traffic-check.sh new file mode 100755 index 0000000..6c1243c --- /dev/null +++ b/test/traffic-check.sh @@ -0,0 +1,359 @@ +#!/bin/bash +# Runs inside a throwaway privileged container (see test/traffic.sh): the +# blocklist and the per-peer shaping the exit node applies to forwarded traffic. +# A veth pair named wggvpn stands in for WireGuard, a second namespace for a VPN +# client, and a third for a public host that answers, so probes are really +# forwarded across wggvpn instead of only checked against a ruleset. +set -euo pipefail + +HOOK=/app/wg-hooks.sh +BLOCKLIST=/app/gvpn-blocklist-update.sh +WG_IF=wggvpn +IFB_IF=ifb-gvpn +BLOCKLIST_SET=gvpn-blocklist +CLIENT_CIDR=10.128.0.0/24 +SERVER_ADDRESS=10.128.0.1 +CLIENT_ADDRESS=10.128.0.2 +# The uplink, and a public host behind it that answers. Both are TEST-NET +# addresses: public as far as the rules are concerned, and never routed. +UPLINK=eth1 +PUBLIC_HOST=203.0.113.9 +PUBLIC_SUBNET=203.0.113.0/24 +# Blocked by the feed below, and not otherwise private, so only the blocklist +# can stop it. +BLOCKED_HOST=203.0.113.77 +CLIENT_NS=vpn-client +CLIENT_IF=veth-client +INTERNET_NS=internet +INTERNET_IF=inet0 +CACHE_DIR=/tmp/blocklist +GOOD_FEED=/tmp/feed-good +SHORT_FEED=/tmp/feed-short +EXTRA_FEED=/tmp/feed-extra + +# What entrypoint.sh derives from WIREGUARD_CIDR and MAX_SLOTS and exports for +# wg-hooks.sh. Rates are the shipped defaults, so tc does not warn about small +# quanta. +export GVPN_PEER_SUBNET_PREFIX=10.128.0 +export GVPN_PEER_MIN_OCTET=2 +export GVPN_PEER_MAX_OCTET=5 +# Deliberately not the shipped defaults, so a rate the setup wizard sets has to +# reach the classes for these checks to pass. +export GVPN_PEER_RATE_KBIT=7000 +export GVPN_PEER_SHAPING_AGGREGATE_KBIT=18000 +export GVPN_BLOCKLIST_CACHE_DIR="${CACHE_DIR}" +export GVPN_BLOCKLIST_MIN_ENTRIES=3 +# Local feeds only: the refresher runs curl, and the sandbox has no route off +# the machine. +export GVPN_BLOCKLIST_FEEDS="file://${GOOD_FEED}" + +failures=0 +pass() { echo "ok: $*"; } +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +setup() { + # A public host behind the uplink that answers pings. + ip netns add "${INTERNET_NS}" + ip link add "${UPLINK}" type veth peer name "${INTERNET_IF}" + ip link set "${INTERNET_IF}" netns "${INTERNET_NS}" + ip addr add 198.51.100.10/24 dev "${UPLINK}" + ip link set "${UPLINK}" up + ip route replace "${PUBLIC_SUBNET}" dev "${UPLINK}" + ip route replace default dev "${UPLINK}" + ip -n "${INTERNET_NS}" addr add 198.51.100.9/24 dev "${INTERNET_IF}" + ip -n "${INTERNET_NS}" addr add "${PUBLIC_HOST}/24" dev "${INTERNET_IF}" + ip -n "${INTERNET_NS}" link set "${INTERNET_IF}" up + ip -n "${INTERNET_NS}" link set lo up + ip -n "${INTERNET_NS}" route add default via 198.51.100.10 + + # The tunnel, and the client behind it. + ip netns add "${CLIENT_NS}" + ip link add "${WG_IF}" type veth peer name "${CLIENT_IF}" + ip link set "${CLIENT_IF}" netns "${CLIENT_NS}" + ip addr add "${SERVER_ADDRESS}/24" dev "${WG_IF}" + ip link set "${WG_IF}" up + ip -n "${CLIENT_NS}" addr add "${CLIENT_ADDRESS}/24" dev "${CLIENT_IF}" + ip -n "${CLIENT_NS}" link set "${CLIENT_IF}" up + ip -n "${CLIENT_NS}" link set lo up + ip -n "${CLIENT_NS}" route add default via "${SERVER_ADDRESS}" + + # Three entries: one that matters (the blocked host) and two that only get + # the feed over the minimum-entry threshold. + cat >"${GOOD_FEED}" <"${SHORT_FEED}" + # A second feed, to check that several feeds are merged. + printf '%s/32\n' 203.0.113.79 >"${EXTRA_FEED}" +} + +# Packet count of the rule in CHAIN whose `iptables -S -v` line contains SPEC, +# with the `-c PACKETS BYTES` counters stripped. Counters are zeroed before +# every probe, so the count belongs to that probe. +counter() { + local chain="$1" spec="$2" + iptables -t filter -S "${chain}" -v | awk -v want="${spec}" -v prefix="-A ${chain} " ' + { + line = $0 + pkts = "missing" + if (match(line, / -c [0-9]+ [0-9]+/)) { + split(substr(line, RSTART + 4, RLENGTH - 4), counts, " ") + pkts = counts[1] + line = substr(line, 1, RSTART - 1) substr(line, RSTART + RLENGTH) + } + # This chain only, so a spec cannot match a rule somewhere else. + if (index(line, prefix) == 1 && index(line, want) > 0) { print pkts; found = 1; exit } + } + END { if (!found) print "missing" } + ' || true +} + +expect_matched() { + local chain="$1" spec="$2" description="$3" pkts + pkts="$(counter "${chain}" "${spec}")" + if [ "${pkts}" = missing ]; then + fail "${description}: no rule '${spec}' in ${chain}" + elif [ "${pkts}" -lt 1 ]; then + fail "${description}: rule '${spec}' in ${chain} matched ${pkts} packets" + else + pass "${description}" + fi +} + +expect_unmatched() { + local chain="$1" spec="$2" description="$3" pkts + pkts="$(counter "${chain}" "${spec}")" + if [ "${pkts}" = missing ]; then + fail "${description}: no rule '${spec}' in ${chain}" + elif [ "${pkts}" -ne 0 ]; then + fail "${description}: rule '${spec}' in ${chain} matched ${pkts} packets" + else + pass "${description}" + fi +} + +# probe_from_client DESTINATION: succeeds when the public host answered. +probe_from_client() { + ip netns exec "${CLIENT_NS}" ping -c 1 -W 1 "$1" >/dev/null 2>&1 +} + +zero_counters() { + local chain + for chain in FORWARD INPUT; do + iptables -t filter -Z "${chain}" + done +} + +# `|| true` so a missing set reports as a failed check instead of aborting the +# run under `set -e`. +blocklist_entries() { + ipset list "${BLOCKLIST_SET}" 2>/dev/null | awk '/Number of entries/ { print $4 }' || true +} + +# expect_entries COUNT DESCRIPTION +expect_entries() { + local expected="$1" description="$2" actual + actual="$(blocklist_entries)" + if [ "${actual}" = "${expected}" ]; then + pass "${description}" + else + fail "${description}: set holds '${actual:-nothing}', expected ${expected}" + fi +} + +# Packets sent through class CLASSID of DEV, from `tc -s class show`. +class_packets() { + tc -s class show dev "$1" classid "$2" | awk '/Sent [0-9]+ bytes/ { for (i = 1; i < NF; i++) if ($i == "bytes") { print $(i + 1); exit } }' || true +} + +expect_class_packets() { + local dev="$1" classid="$2" description="$3" pkts + pkts="$(class_packets "${dev}" "${classid}")" + if [ -z "${pkts}" ]; then + fail "${description}: no class ${classid} on ${dev}" + elif [ "${pkts}" -lt 1 ]; then + fail "${description}: class ${classid} on ${dev} saw ${pkts} packets" + else + pass "${description}" + fi +} + +setup + +# --- the hook installs the blocklist drops and the shaping ------------------- +"${HOOK}" up "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + +if ipset list "${BLOCKLIST_SET}" >/dev/null 2>&1; then + pass "the hook creates ${BLOCKLIST_SET}, so its DROP rules resolve" +else + fail "the hook did not create ${BLOCKLIST_SET}" +fi + +expect_entries 0 "an empty blocklist blocks nothing" + +# Both directions, and both ahead of the ACCEPT that lets clients out. +# `|| true` throughout: a missing rule must report as a failed check rather +# than abort the run, so a regression shows all of its symptoms at once. +blocked_rule="$(iptables -t filter -S FORWARD | grep -nF -- "-m set --match-set ${BLOCKLIST_SET} dst -j DROP" | cut -d: -f1 || true)" +accepted_rule="$(iptables -t filter -S FORWARD | grep -nF -- "-i ${WG_IF} -o ${UPLINK} -j ACCEPT" | cut -d: -f1 || true)" +if [ -n "${blocked_rule}" ] && [ -n "${accepted_rule}" ] && [ "${blocked_rule}" -lt "${accepted_rule}" ]; then + pass "the client -> blocklisted destination drop precedes the client ACCEPT" +else + fail "the client -> blocklisted destination drop is missing or behind the client ACCEPT" +fi +if iptables -t filter -S FORWARD | grep -qF -- "-o ${WG_IF} -m set --match-set ${BLOCKLIST_SET} src -j DROP"; then + pass "blocklisted sources are dropped on their way back to clients" +else + fail "no FORWARD drop for blocklisted sources returning to clients" +fi + +# --- a refresh replaces the set, atomically, and the cache follows ------------ +if "${BLOCKLIST}" --refresh >/dev/null 2>&1; then + pass "--refresh exits 0" +else + fail "--refresh failed" +fi +expect_entries 3 "a good feed populates the set" +if [ -s "${CACHE_DIR}/blocklist.cidr" ]; then + pass "a good feed is cached for the next boot" +else + fail "no cache file after a successful refresh" +fi + +# --- several feeds are merged, space- or newline-separated ------------------- +GVPN_BLOCKLIST_FEEDS="file://${GOOD_FEED} file://${EXTRA_FEED}" "${BLOCKLIST}" --refresh >/dev/null 2>&1 || true +expect_entries 4 "space-separated feeds are merged" +GVPN_BLOCKLIST_FEEDS="$(printf 'file://%s\nfile://%s' "${GOOD_FEED}" "${EXTRA_FEED}")" \ + "${BLOCKLIST}" --refresh >/dev/null 2>&1 || true +expect_entries 4 "newline-separated feeds are merged" +"${BLOCKLIST}" --refresh >/dev/null 2>&1 || true +expect_entries 3 "the configured feeds replace the merged set" + +zero_counters +if probe_from_client "${BLOCKED_HOST}"; then + fail "the client reached a blocklisted destination" +else + pass "the client cannot reach a blocklisted destination" +fi +expect_matched FORWARD "-m set --match-set ${BLOCKLIST_SET} dst -j DROP" "the blocklist drop matched it" +expect_unmatched FORWARD "-i ${WG_IF} -o ${UPLINK} -j ACCEPT" "the blocklisted packet was not forwarded" + +# --- a feed that is too short is rejected, the previous set stays ------------- +GVPN_BLOCKLIST_FEEDS="file://${SHORT_FEED}" "${BLOCKLIST}" --refresh >/dev/null 2>&1 || true +expect_entries 3 "a feed below the minimum entry count keeps the previous set" + +zero_counters +probe_from_client 203.0.113.78 >/dev/null 2>&1 || true +expect_unmatched FORWARD "-m set --match-set ${BLOCKLIST_SET} dst -j DROP" "the rejected feed's entry is not blocked" + +# --- an unreachable feed falls back to the cache, not to an empty set -------- +GVPN_BLOCKLIST_FEEDS="file:///tmp/does-not-exist" "${BLOCKLIST}" --refresh >/dev/null 2>&1 || true +expect_entries 3 "an unreachable feed falls back to the cached list" + +# --- the cache reloads the set without the network --------------------------- +ipset flush "${BLOCKLIST_SET}" +expect_entries 0 "the set is empty after a flush" +"${BLOCKLIST}" --from-cache >/dev/null 2>&1 || true +expect_entries 3 "--from-cache reloads the cached list" + +# --- short prefixes are accepted for the 224/3 bogon range only --------------- +# ipset normalizes a network, so 224.0.0.0/1 would block 128.0.0.0/1, and a /0 +# fails the whole restore. +parsed="$(printf '%s\n' 224.0.0.0/3 224.0.0.0/1 240.0.0.0/2 240.0.0.0/0 0.0.0.0/1 1.2.3.0/24 \ + | bash -c "source ${BLOCKLIST}; parse_feed" || true)" +if [ "${parsed}" = "$(printf '%s\n' 224.0.0.0/3 1.2.3.0/24)" ]; then + pass "the parser keeps 224.0.0.0/3 and drops every other prefix shorter than /8" +else + fail "the parser kept: ${parsed//$'\n'/ }" +fi + +# --- repeated refreshes in one process leave no temp files -------------------- +# --loop never exits, so its EXIT trap cannot be what cleans up after a refresh. +tmp_root="$(mktemp -d)" +TMPDIR="${tmp_root}" bash -c "source ${BLOCKLIST}; refresh; refresh" >/dev/null 2>&1 || true +leftover="$(find "${tmp_root}" -mindepth 1 | wc -l)" +if [ "${leftover}" -eq 0 ]; then + pass "each refresh removes its temp files" +else + fail "${leftover} temp file(s) left behind by two refreshes" +fi +rm -rf "${tmp_root}" + +# --- a client still reaches public destinations ------------------------------ +zero_counters +if probe_from_client "${PUBLIC_HOST}"; then + pass "the client still reaches a public host" +else + fail "the client could not reach a public host" +fi +expect_unmatched FORWARD "-m set --match-set ${BLOCKLIST_SET} dst -j DROP" "the public host is not blocklisted" + +# --- every peer has a shaped class in both directions ------------------------ +class_count="$(tc class show dev "${WG_IF}" | grep -c 'class htb 1:' || true)" +if [ "${class_count}" -eq 6 ]; then + pass "one class per peer, plus the aggregate and the default class" +else + fail "${class_count} classes on ${WG_IF}, expected 6 (aggregate, default, 4 peers)" +fi +if tc class show dev "${WG_IF}" classid 1:1 | grep -q 'rate 18Mbit'; then + pass "the aggregate class caps the node-wide rate" +else + fail "the aggregate class is not capped at 18Mbit" +fi +if tc class show dev "${WG_IF}" classid 1:2 | grep -q 'ceil 7Mbit'; then + pass "each peer class is capped at the per-peer rate" +else + fail "peer class 1:2 is not capped at 7Mbit" +fi +if [ "$(tc qdisc show dev "${WG_IF}" | grep -c fq_codel)" -eq 5 ]; then + pass "every class has an fq_codel leaf" +else + fail "fq_codel leaves are missing from ${WG_IF}" +fi +if ip link show "${IFB_IF}" >/dev/null 2>&1 && tc class show dev "${IFB_IF}" classid 1:2 | grep -q 'ceil 7Mbit'; then + pass "uploads are shaped on ${IFB_IF}, mirrored from ${WG_IF}" +else + fail "no shaped ${IFB_IF} for upload traffic" +fi + +# The probes above already moved packets through peer .2: the reply left via +# wggvpn (classified by destination) and the request arrived on the mirror +# (classified by source). +expect_class_packets "${WG_IF}" 1:2 "download traffic to peer .2 is shaped" +expect_class_packets "${IFB_IF}" 1:2 "upload traffic from peer .2 is shaped" + +# --- down removes the drops and the shaping, and keeps the set -------------- +"${HOOK}" down "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + +if iptables -t filter -S FORWARD | grep -qF -- "-m set --match-set ${BLOCKLIST_SET}"; then + fail "FORWARD still references ${BLOCKLIST_SET} after down" +else + pass "down removes the blocklist drops" +fi +expect_entries 3 "down keeps the blocklist set, so the refresher keeps working" +if tc qdisc show dev "${WG_IF}" | grep -q 'htb'; then + fail "the shaping tree on ${WG_IF} survived down" +else + pass "down removes the shaping tree" +fi +if ip link show "${IFB_IF}" >/dev/null 2>&1; then + fail "${IFB_IF} survived down" +else + pass "down removes ${IFB_IF}" +fi + +if [ "${failures}" -ne 0 ]; then + echo "--- final state ---" >&2 + iptables -t filter -S >&2 + ipset list "${BLOCKLIST_SET}" >&2 2>/dev/null || true + tc qdisc show dev "${WG_IF}" >&2 + echo "${failures} check(s) failed" >&2 + exit 1 +fi +echo "all traffic checks passed" diff --git a/test/traffic.sh b/test/traffic.sh new file mode 100755 index 0000000..6239bae --- /dev/null +++ b/test/traffic.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Checks the blocklist and the per-peer shaping the exit node applies to +# forwarded traffic: first that the image ships both scripts and wires them up, +# then how they behave in a throwaway privileged container (see +# test/traffic-check.sh). +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +# wg-hooks.sh runs the shaping script from PostUp/PreDown, and entrypoint.sh +# loads the cached blocklist before the server starts and refreshes it in the +# background. An image that stopped shipping either would fail here. +grep -q '^COPY .*gvpn-blocklist-update.sh' gnosisvpn-server/Dockerfile +grep -q '^COPY .*gvpn-peer-shaping.sh' gnosisvpn-server/Dockerfile +grep -q 'gvpn-blocklist-update.sh --from-cache' gnosisvpn-server/entrypoint.sh +grep -q 'gvpn-blocklist-update.sh --loop &' gnosisvpn-server/entrypoint.sh +grep -q 'GVPN_SHAPING_BIN:-/app/gvpn-peer-shaping.sh' gnosisvpn-server/wg-hooks.sh + +# The rates and the feeds are set from the DAppNode UI, so their setup-wizard +# fields have to exist. +for field in GVPN_PEER_RATE_KBIT GVPN_PEER_SHAPING_AGGREGATE_KBIT \ + GVPN_BLOCKLIST_FEEDS GVPN_BLOCKLIST_REFRESH_INTERVAL_S; do + if ! grep -qx " name: ${field}" setup-wizard.yml; then + echo "setup-wizard.yml has no ${field} field" >&2 + exit 1 + fi +done +echo "ok: the image ships both scripts, wires them up, and the wizard exposes their settings" + +if [ -n "${TRAFFIC_TEST_IMAGE:-}" ]; then + # Tests the image's own /app, so nothing is mounted or installed. --entrypoint + # skips entrypoint.sh, which would start the server. + docker run --rm --privileged --entrypoint bash \ + -v "${PWD}/test/traffic-check.sh:/traffic-check.sh:ro" \ + "${TRAFFIC_TEST_IMAGE}" \ + /traffic-check.sh +else + docker run --rm --privileged \ + -v "${PWD}/gnosisvpn-server:/app:ro" \ + -v "${PWD}/test/traffic-check.sh:/traffic-check.sh:ro" \ + alpine:3 \ + sh -c 'apk add --no-cache --quiet bash iptables iproute2 ipset curl iputils >/dev/null && bash /traffic-check.sh' +fi diff --git a/test/validate-hoprd-config.sh b/test/validate-hoprd-config.sh index a9be0c1..8385207 100755 --- a/test/validate-hoprd-config.sh +++ b/test/validate-hoprd-config.sh @@ -3,10 +3,8 @@ # (`hoprd-cfg --validate-args`), the same check the container's entrypoint effectively runs # (via hoprd itself) before launching. Keep HOPRD_IMAGE's tag in sync with docker-compose.yml's # UPSTREAM_VERSION - both deliberately track hoprd's `latest` release (no version/digest pin), -# so if one is ever re-pinned to a specific version, re-pin the other too. Renders the -# template with a dummy IP first - entrypoint.sh does the same substitution with -# gnosisvpn-server's real resolved IP - since hoprd's parser requires a literal ip:port and -# won't accept the __GNOSISVPN_SERVER_IP__ placeholder as-is. +# so if one is ever re-pinned to a specific version, re-pin the other too. Fills in the +# placeholders first, as entrypoint.sh does. set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." @@ -15,7 +13,9 @@ HOPRD_IMAGE="europe-west3-docker.pkg.dev/hoprassociation/docker-images/hoprd:${H RENDERED_CONFIG="$(mktemp)" trap 'rm -f "${RENDERED_CONFIG}"' EXIT -sed "s|__GNOSISVPN_SERVER_IP__|127.0.0.1|g" hoprd.cfg.yaml.tpl >"${RENDERED_CONFIG}" +BLOKLI_URL="$(sed -n 's|^blokli_url: ||p' hoprd.cfg.yaml.tpl)" +sed -e "s|__GNOSISVPN_SERVER_IP__|172.33.255.254|g" -e "s|__BLOKLI_URL__|${BLOKLI_URL}|g" \ + hoprd.cfg.yaml.tpl >"${RENDERED_CONFIG}" docker pull "${HOPRD_IMAGE}" >/dev/null docker run --rm \ -v "${RENDERED_CONFIG}:/app/hoprd.cfg.yaml:ro" \ @@ -31,4 +31,4 @@ docker run --rm \ --host 1.2.3.4:9091 \ --configurationFilePath /app/hoprd.cfg.yaml -echo "hoprd.cfg.yaml.tpl is a valid config for ${HOPRD_IMAGE} (rendered with a dummy IP)." +echo "hoprd.cfg.yaml.tpl is a valid config for ${HOPRD_IMAGE}."