mirror of
https://github.com/shadowsocks/ShadowsocksX-NG.git
synced 2026-11-22 13:34:00 +00:00
169 lines
6.0 KiB
Bash
Executable File
169 lines
6.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Explicit maintainer update for the built-in GFWList AutoProxy rule snapshot
|
|
# (issue #65). Normal builds never run this script — they only read the local
|
|
# pinned snapshot. On any failure the previous valid snapshot is kept.
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
RULES_DIR="${ROOT}/Vendor/rules/gfwlist"
|
|
SNAPSHOT="${RULES_DIR}/snapshot.json"
|
|
MANIFEST="${RULES_DIR}/manifest.json"
|
|
NOTICE="${RULES_DIR}/NOTICE"
|
|
|
|
# Pinned upstream: gfwlist/gfwlist master commit. Bump these together after
|
|
# human review of the new commit.
|
|
UPSTREAM_VERSION="c19292511da4c5af8f6193fd32d666eaaa0312f9"
|
|
UPSTREAM_URL="https://raw.githubusercontent.com/gfwlist/gfwlist/${UPSTREAM_VERSION}/gfwlist.txt"
|
|
UPSTREAM_MIRROR_URL="https://cdn.jsdelivr.net/gh/gfwlist/gfwlist@${UPSTREAM_VERSION}/gfwlist.txt"
|
|
UPSTREAM_SHA256="750d34564363075fafa12d8ab9739f269e2fbd2bf3c5f4ab8cfd859bfd3441df"
|
|
LICENSE="LGPL-2.1"
|
|
ATTRIBUTION="GFWList AutoProxy rules derived from gfwlist/gfwlist gfwlist.txt at commit ${UPSTREAM_VERSION}. Official Base64 AutoProxy 0.2.9 list. Only rules losslessly expressible as target domains are converted; URL-path, protocol, wildcard, and regex conditions are reported as losses and never expanded into whole-domain rules. List is candidate routing data, not a guarantee of reachability or completeness."
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: Scripts/update-gfwlist.sh [--fetch-only]
|
|
|
|
Explicitly refresh Vendor/rules/gfwlist/snapshot.json from the pinned
|
|
gfwlist/gfwlist commit. This is a maintainer action; ordinary builds are
|
|
offline and only read the local snapshot.
|
|
|
|
--fetch-only Download and verify gfwlist.txt without converting.
|
|
EOF
|
|
}
|
|
|
|
FETCH_ONLY=0
|
|
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
|
usage
|
|
exit 0
|
|
fi
|
|
if [[ "${1:-}" == "--fetch-only" ]]; then
|
|
FETCH_ONLY=1
|
|
elif [[ $# -gt 0 ]]; then
|
|
usage >&2
|
|
exit 2
|
|
fi
|
|
|
|
TMP="$(mktemp -d)"
|
|
cleanup() { rm -rf "${TMP}"; }
|
|
trap cleanup EXIT
|
|
|
|
echo "Fetching gfwlist.txt ${UPSTREAM_VERSION} …"
|
|
fetch_ok=0
|
|
if command -v curl >/dev/null 2>&1; then
|
|
if curl -fsSL --retry 1 --max-time 20 -o "${TMP}/gfwlist.txt" "${UPSTREAM_URL}"; then
|
|
fetch_ok=1
|
|
echo "Fetched from ${UPSTREAM_URL}"
|
|
elif curl -fsSL --retry 2 --max-time 45 -o "${TMP}/gfwlist.txt" "${UPSTREAM_MIRROR_URL}"; then
|
|
fetch_ok=1
|
|
echo "Fetched from mirror ${UPSTREAM_MIRROR_URL}"
|
|
fi
|
|
fi
|
|
if [[ "${fetch_ok}" -ne 1 ]]; then
|
|
echo "error: failed to fetch gfwlist.txt from pinned upstream or mirror" >&2
|
|
echo "previous snapshot kept"
|
|
exit 1
|
|
fi
|
|
|
|
ACTUAL="$(shasum -a 256 "${TMP}/gfwlist.txt" | awk '{print $1}')"
|
|
if [[ "${ACTUAL}" != "${UPSTREAM_SHA256}" ]]; then
|
|
echo "error: gfwlist.txt sha256 mismatch" >&2
|
|
echo " expected ${UPSTREAM_SHA256}" >&2
|
|
echo " actual ${ACTUAL}" >&2
|
|
echo "previous snapshot kept"
|
|
exit 1
|
|
fi
|
|
echo "Verified gfwlist.txt sha256"
|
|
|
|
if [[ "${FETCH_ONLY}" -eq 1 ]]; then
|
|
echo "fetch-only: leaving snapshot untouched"
|
|
exit 0
|
|
fi
|
|
|
|
mkdir -p "${RULES_DIR}"
|
|
|
|
# Previous rule count feeds scale-change detection; missing snapshot is first run.
|
|
PREVIOUS_COUNT=""
|
|
if [[ -f "${SNAPSHOT}" ]]; then
|
|
PREVIOUS_COUNT="$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))["rules"]))' "${SNAPSHOT}")"
|
|
fi
|
|
|
|
CONVERT_ARGS=(
|
|
"${TMP}/gfwlist.txt"
|
|
--upstream "${UPSTREAM_VERSION}"
|
|
--upstream-reference "gfwlist/gfwlist gfwlist.txt @ ${UPSTREAM_VERSION}"
|
|
--license "${LICENSE}"
|
|
--attribution "${ATTRIBUTION}"
|
|
--out "${TMP}/snapshot.json"
|
|
)
|
|
if [[ -n "${PREVIOUS_COUNT}" ]]; then
|
|
CONVERT_ARGS+=(--previous-count "${PREVIOUS_COUNT}")
|
|
fi
|
|
|
|
# Convert into a temporary snapshot first; only replace on success.
|
|
if ! python3 "${ROOT}/Scripts/convert-gfwlist.py" "${CONVERT_ARGS[@]}"; then
|
|
echo "error: conversion failed; previous snapshot kept"
|
|
exit 1
|
|
fi
|
|
|
|
# Sanity: snapshot must load as JSON and carry required metadata keys.
|
|
python3 - "${TMP}/snapshot.json" <<'PY'
|
|
import json, sys
|
|
from pathlib import Path
|
|
data = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
|
|
meta = data.get("metadata") or {}
|
|
for key in ("source", "upstreamReference", "inputDigest", "fetchedAt", "converterVersion", "license", "attribution"):
|
|
if key not in meta:
|
|
raise SystemExit(f"snapshot metadata missing {key}")
|
|
if not data.get("rules"):
|
|
raise SystemExit("snapshot has no rules")
|
|
print(f"snapshot ok: {len(data['rules'])} rules, "
|
|
f"shadowed={data.get('lossReport', {}).get('skipped', {}).get('shadowedException', 0)}")
|
|
PY
|
|
|
|
cp "${TMP}/snapshot.json" "${SNAPSHOT}"
|
|
|
|
# Refresh manifest (build-time integrity anchor).
|
|
python3 - "${SNAPSHOT}" "${MANIFEST}" "${UPSTREAM_VERSION}" "${UPSTREAM_URL}" "${UPSTREAM_SHA256}" <<'PY'
|
|
import hashlib, json, sys
|
|
from pathlib import Path
|
|
snapshot_path, manifest_path, version, url, sha256 = sys.argv[1:6]
|
|
data = Path(snapshot_path).read_bytes()
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
manifest = {
|
|
"name": "gfwlist",
|
|
"upstreamVersion": version,
|
|
"upstreamURL": url,
|
|
"upstreamArchiveSHA256": sha256,
|
|
"snapshotSHA256": digest,
|
|
"converterVersion": json.loads(data)["metadata"]["converterVersion"],
|
|
}
|
|
Path(manifest_path).write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(f"wrote {manifest_path}")
|
|
PY
|
|
|
|
# NOTICE: license and attribution accompany the distributed snapshot.
|
|
cat > "${NOTICE}" <<EOF
|
|
gfwlist rule snapshot
|
|
=====================
|
|
|
|
Upstream: gfwlist/gfwlist gfwlist.txt
|
|
Upstream commit: ${UPSTREAM_VERSION}
|
|
Upstream URL: ${UPSTREAM_URL}
|
|
Upstream archive SHA-256: ${UPSTREAM_SHA256}
|
|
|
|
License: ${LICENSE}
|
|
|
|
${ATTRIBUTION}
|
|
|
|
This list is candidate routing data. It is not a guarantee of network
|
|
reachability, geographic ownership, or privacy. Rules that cannot be expressed
|
|
losslessly as target domains (URL paths, protocol conditions, wildcards, regex)
|
|
are counted and reported as conversion losses and are never expanded into
|
|
whole-domain proxy rules. \`@@\` exceptions shadowed by a broader proxy rule are
|
|
reported item by item and omitted from the generated ACL because sslocal would
|
|
ignore them.
|
|
EOF
|
|
|
|
echo "Updated ${SNAPSHOT}"
|
|
echo "Remember to review the diff and commit the snapshot together with its NOTICE and manifest."
|