#!/usr/bin/env bash # Explicit maintainer update for the built-in GFWList AutoProxy rule snapshot # Normal builds never run this script — they only read the local # pinned snapshot. On any failure the previous valid snapshot is kept. set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" RULES_DIR="${ROOT}/Vendor/rules/gfwlist" SNAPSHOT="${RULES_DIR}/snapshot.json" MANIFEST="${RULES_DIR}/manifest.json" NOTICE="${RULES_DIR}/NOTICE" # Pinned upstream: edit the manifest after human review of the new commit. UPSTREAM_VERSION="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["upstreamVersion"])' "$MANIFEST")" UPSTREAM_URL="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["upstreamURL"])' "$MANIFEST")" UPSTREAM_SHA256="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["upstreamArchiveSHA256"])' "$MANIFEST")" UPSTREAM_MIRROR_URL="https://cdn.jsdelivr.net/gh/gfwlist/gfwlist@${UPSTREAM_VERSION}/gfwlist.txt" LICENSE="LGPL-2.1" ATTRIBUTION="GFWList AutoProxy rules derived from gfwlist/gfwlist gfwlist.txt at commit ${UPSTREAM_VERSION}. Official Base64 AutoProxy 0.2.9 list. Domain anchors become suffix matches. URL prefixes with an empty or root path, no query, fragment or user information, and a literal domain host become exact matches, discarding scheme and any valid port. Other URL conditions, wildcard hosts, IP literals and regex rules are reported as losses. List is candidate routing data, not a guarantee of reachability or completeness." usage() { cat <<'EOF' Usage: Scripts/update-gfwlist.sh [--fetch-only] Explicitly refresh Vendor/rules/gfwlist/snapshot.json from the pinned gfwlist/gfwlist commit. This is a maintainer action; ordinary builds are offline and only read the local snapshot. --fetch-only Download and verify gfwlist.txt without converting. EOF } FETCH_ONLY=0 if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then usage exit 0 fi if [[ "${1:-}" == "--fetch-only" ]]; then FETCH_ONLY=1 elif [[ $# -gt 0 ]]; then usage >&2 exit 2 fi TMP="$(mktemp -d)" cleanup() { rm -rf "${TMP}"; } trap cleanup EXIT echo "Fetching gfwlist.txt ${UPSTREAM_VERSION} …" fetch_ok=0 if command -v curl >/dev/null 2>&1; then if curl -fsSL --retry 1 --max-time 20 -o "${TMP}/gfwlist.txt" "${UPSTREAM_URL}"; then fetch_ok=1 echo "Fetched from ${UPSTREAM_URL}" elif curl -fsSL --retry 2 --max-time 45 -o "${TMP}/gfwlist.txt" "${UPSTREAM_MIRROR_URL}"; then fetch_ok=1 echo "Fetched from mirror ${UPSTREAM_MIRROR_URL}" fi fi if [[ "${fetch_ok}" -ne 1 ]]; then echo "error: failed to fetch gfwlist.txt from pinned upstream or mirror" >&2 echo "previous snapshot kept" exit 1 fi ACTUAL="$(shasum -a 256 "${TMP}/gfwlist.txt" | awk '{print $1}')" if [[ "${ACTUAL}" != "${UPSTREAM_SHA256}" ]]; then echo "error: gfwlist.txt sha256 mismatch" >&2 echo " expected ${UPSTREAM_SHA256}" >&2 echo " actual ${ACTUAL}" >&2 echo "previous snapshot kept" exit 1 fi echo "Verified gfwlist.txt sha256" if [[ "${FETCH_ONLY}" -eq 1 ]]; then echo "fetch-only: leaving snapshot untouched" exit 0 fi mkdir -p "${RULES_DIR}" # Previous rule count feeds scale-change detection; missing snapshot is first run. PREVIOUS_COUNT="" if [[ -f "${SNAPSHOT}" ]]; then PREVIOUS_COUNT="$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))["rules"]))' "${SNAPSHOT}")" fi CONVERT_ARGS=( "${TMP}/gfwlist.txt" --upstream "${UPSTREAM_VERSION}" --upstream-reference "gfwlist/gfwlist gfwlist.txt @ ${UPSTREAM_VERSION}" --license "${LICENSE}" --attribution "${ATTRIBUTION}" --out "${TMP}/snapshot.json" ) if [[ -n "${PREVIOUS_COUNT}" ]]; then CONVERT_ARGS+=(--previous-count "${PREVIOUS_COUNT}") fi # Convert into a temporary snapshot first; only replace on success. if ! python3 "${ROOT}/Scripts/convert-gfwlist.py" "${CONVERT_ARGS[@]}"; then echo "error: conversion failed; previous snapshot kept" exit 1 fi # Sanity: snapshot must load as JSON and carry required metadata keys. python3 - "${TMP}/snapshot.json" <<'PY' import json, sys from pathlib import Path data = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) meta = data.get("metadata") or {} for key in ("source", "upstreamReference", "inputDigest", "fetchedAt", "converterVersion", "license", "attribution"): if key not in meta: raise SystemExit(f"snapshot metadata missing {key}") if not data.get("rules"): raise SystemExit("snapshot has no rules") print(f"snapshot ok: {len(data['rules'])} rules, " f"shadowed={data.get('lossReport', {}).get('absorbedCount', 0)}") PY cp "${TMP}/snapshot.json" "${SNAPSHOT}" # Refresh manifest (build-time integrity anchor). python3 - "${SNAPSHOT}" "${MANIFEST}" "${UPSTREAM_VERSION}" "${UPSTREAM_URL}" "${UPSTREAM_SHA256}" <<'PY' import hashlib, json, sys from pathlib import Path snapshot_path, manifest_path, version, url, sha256 = sys.argv[1:6] data = Path(snapshot_path).read_bytes() digest = hashlib.sha256(data).hexdigest() manifest = { "name": "gfwlist", "upstreamVersion": version, "upstreamURL": url, "upstreamArchiveSHA256": sha256, "snapshotSHA256": digest, "converterVersion": json.loads(data)["metadata"]["converterVersion"], } Path(manifest_path).write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") print(f"wrote {manifest_path}") PY # NOTICE: license and attribution accompany the distributed snapshot. cat > "${NOTICE}" <