auto update gfwlist, see https://github.com/shadowsocks/shadowsocks-csharp/issues/115
This commit is contained in:
Gang Zhuo
@@ -0,0 +1,283 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Net;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Security.Cryptography;
|
||||
using Shadowsocks.Model;
|
||||
using Shadowsocks.Properties;
|
||||
|
||||
namespace Shadowsocks.Controller
|
||||
{
|
||||
public class GfwListUpdater
|
||||
{
|
||||
private const string GFWLIST_URL = "https://autoproxy-gfwlist.googlecode.com/svn/trunk/gfwlist.txt";
|
||||
|
||||
private const int EXPIRE_HOURS = 6;
|
||||
|
||||
public IWebProxy proxy = null;
|
||||
|
||||
public bool useSystemProxy = true;
|
||||
|
||||
public class GfwListChangedArgs : EventArgs
|
||||
{
|
||||
public string[] GfwList { get; set; }
|
||||
}
|
||||
|
||||
public event EventHandler<GfwListChangedArgs> GfwListChanged;
|
||||
|
||||
private bool running = false;
|
||||
private bool closed = false;
|
||||
private int jobId = 0;
|
||||
DateTime lastUpdateTimeUtc;
|
||||
string lastUpdateMd5;
|
||||
|
||||
private object locker = new object();
|
||||
|
||||
public GfwListUpdater()
|
||||
{
|
||||
}
|
||||
|
||||
~GfwListUpdater()
|
||||
{
|
||||
Stop();
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
lock (locker)
|
||||
{
|
||||
if (running)
|
||||
return;
|
||||
running = true;
|
||||
closed = false;
|
||||
jobId++;
|
||||
new Thread(new ParameterizedThreadStart(UpdateJob)).Start(jobId);
|
||||
}
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
lock(locker)
|
||||
{
|
||||
closed = true;
|
||||
running = false;
|
||||
jobId++;
|
||||
}
|
||||
}
|
||||
|
||||
public void ScheduleUpdateTime(int delaySeconds)
|
||||
{
|
||||
lock(locker)
|
||||
{
|
||||
lastUpdateTimeUtc = DateTime.UtcNow.AddHours(-1 * EXPIRE_HOURS).AddSeconds(delaySeconds);
|
||||
}
|
||||
}
|
||||
|
||||
private string DownloadGfwListFile()
|
||||
{
|
||||
try
|
||||
{
|
||||
WebClient http = new WebClient();
|
||||
http.Proxy = useSystemProxy ? WebRequest.GetSystemWebProxy() : proxy;
|
||||
return http.DownloadString(new Uri(GFWLIST_URL));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine(ex.Message);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private bool IsExpire()
|
||||
{
|
||||
lock (locker)
|
||||
{
|
||||
TimeSpan ts = DateTime.UtcNow - lastUpdateTimeUtc;
|
||||
bool expire = ((int)ts.TotalHours) >= EXPIRE_HOURS;
|
||||
if (expire)
|
||||
lastUpdateTimeUtc = DateTime.UtcNow;
|
||||
return expire;
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsJobStop(int currentJobId)
|
||||
{
|
||||
lock (locker)
|
||||
{
|
||||
if (!running || closed || currentJobId != this.jobId)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private bool IsGfwListChanged(string content)
|
||||
{
|
||||
byte[] inputBytes = Encoding.UTF8.GetBytes(content);
|
||||
byte[] md5Bytes = MD5.Create().ComputeHash(inputBytes);
|
||||
string md5 = "";
|
||||
for (int i = 0; i < md5Bytes.Length; i++)
|
||||
md5 += md5Bytes[i].ToString("x").PadLeft(2, '0');
|
||||
if (md5 == lastUpdateMd5)
|
||||
return false;
|
||||
lastUpdateMd5 = md5;
|
||||
return true;
|
||||
}
|
||||
|
||||
private void ParseGfwList(string response)
|
||||
{
|
||||
if (!IsGfwListChanged(response))
|
||||
return;
|
||||
if (GfwListChanged != null)
|
||||
{
|
||||
Parser parser = new Parser(response);
|
||||
GfwListChangedArgs args = new GfwListChangedArgs {
|
||||
GfwList = parser.GetReducedDomains()
|
||||
};
|
||||
GfwListChanged(this, args);
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateJob(object state)
|
||||
{
|
||||
int currentJobId = (int)state;
|
||||
int retryTimes = 3;
|
||||
while (!IsJobStop(currentJobId))
|
||||
{
|
||||
if (IsExpire())
|
||||
{
|
||||
string response = DownloadGfwListFile();
|
||||
if (response != null)
|
||||
{
|
||||
ParseGfwList(response);
|
||||
}
|
||||
else if (retryTimes > 0)
|
||||
{
|
||||
ScheduleUpdateTime(30); /*Delay 30 seconds to retry*/
|
||||
retryTimes--;
|
||||
}
|
||||
else
|
||||
{
|
||||
retryTimes = 3; /* reset retry times, and wait next update time. */
|
||||
}
|
||||
}
|
||||
|
||||
Thread.Sleep(1000);
|
||||
}
|
||||
}
|
||||
|
||||
class Parser
|
||||
{
|
||||
public string Content { get; private set; }
|
||||
|
||||
public Parser(string response)
|
||||
{
|
||||
byte[] bytes = Convert.FromBase64String(response);
|
||||
this.Content = Encoding.ASCII.GetString(bytes);
|
||||
}
|
||||
|
||||
public string[] GetLines()
|
||||
{
|
||||
return Content.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
}
|
||||
|
||||
/* refer https://github.com/clowwindy/gfwlist2pac/blob/master/gfwlist2pac/main.py */
|
||||
public string[] GetDomains()
|
||||
{
|
||||
string[] lines = GetLines();
|
||||
List<string> domains = new List<string>(lines.Length);
|
||||
for(int i =0;i < lines.Length;i++)
|
||||
{
|
||||
string line = lines[i];
|
||||
if (line.IndexOf(".*") >= 0)
|
||||
continue;
|
||||
else if (line.IndexOf("*") >= 0)
|
||||
line = line.Replace("*", "/");
|
||||
if (line.StartsWith("||"))
|
||||
line = line.Substring(2);
|
||||
else if (line.StartsWith("|"))
|
||||
line = line.Substring(1);
|
||||
else if (line.StartsWith("."))
|
||||
line = line.Substring(1);
|
||||
if (line.StartsWith("!"))
|
||||
continue;
|
||||
else if (line.StartsWith("["))
|
||||
continue;
|
||||
else if (line.StartsWith("@"))
|
||||
continue; /*ignore white list*/
|
||||
domains.Add(line);
|
||||
}
|
||||
return domains.ToArray();
|
||||
}
|
||||
|
||||
/* refer https://github.com/clowwindy/gfwlist2pac/blob/master/gfwlist2pac/main.py */
|
||||
public string[] GetReducedDomains()
|
||||
{
|
||||
string[] domains = GetDomains();
|
||||
List<string> new_domains = new List<string>(domains.Length);
|
||||
IDictionary<string, string> tld_dic = GetTldDictionary();
|
||||
|
||||
foreach(string domain in domains)
|
||||
{
|
||||
string last_root_domain = null;
|
||||
int pos;
|
||||
pos = domain.LastIndexOf('.');
|
||||
last_root_domain = domain.Substring(pos + 1);
|
||||
if (!tld_dic.ContainsKey(last_root_domain))
|
||||
continue;
|
||||
while(pos > 0)
|
||||
{
|
||||
pos = domain.LastIndexOf('.', pos - 1);
|
||||
last_root_domain = domain.Substring(pos + 1);
|
||||
if (tld_dic.ContainsKey(last_root_domain))
|
||||
continue;
|
||||
else
|
||||
break;
|
||||
}
|
||||
if (last_root_domain != null)
|
||||
new_domains.Add(last_root_domain);
|
||||
}
|
||||
|
||||
return new_domains.ToArray();
|
||||
}
|
||||
|
||||
private string[] GetTlds()
|
||||
{
|
||||
string[] tlds = null;
|
||||
byte[] pacGZ = Resources.tld_txt;
|
||||
byte[] buffer = new byte[1024];
|
||||
int n;
|
||||
using(MemoryStream sb = new MemoryStream())
|
||||
{
|
||||
using (GZipStream input = new GZipStream(new MemoryStream(pacGZ),
|
||||
CompressionMode.Decompress, false))
|
||||
{
|
||||
while ((n = input.Read(buffer, 0, buffer.Length)) > 0)
|
||||
{
|
||||
sb.Write(buffer, 0, n);
|
||||
}
|
||||
}
|
||||
tlds = System.Text.Encoding.UTF8.GetString(sb.ToArray())
|
||||
.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
}
|
||||
return tlds;
|
||||
}
|
||||
|
||||
private IDictionary<string, string> GetTldDictionary()
|
||||
{
|
||||
string[] tlds = GetTlds();
|
||||
IDictionary<string, string> dic = new Dictionary<string, string>(tlds.Length);
|
||||
foreach (string tld in tlds)
|
||||
{
|
||||
if (!dic.ContainsKey(tld))
|
||||
dic.Add(tld, tld);
|
||||
}
|
||||
return dic;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ using System.IO.Compression;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace Shadowsocks.Controller
|
||||
{
|
||||
@@ -20,6 +21,8 @@ namespace Shadowsocks.Controller
|
||||
Socket _listener;
|
||||
FileSystemWatcher watcher;
|
||||
|
||||
GfwListUpdater gfwlistUpdater;
|
||||
|
||||
public event EventHandler PACFileChanged;
|
||||
|
||||
public void Start(Configuration configuration)
|
||||
@@ -48,6 +51,7 @@ namespace Shadowsocks.Controller
|
||||
_listener);
|
||||
|
||||
WatchPacFile();
|
||||
StartGfwListUpdater();
|
||||
}
|
||||
catch (SocketException)
|
||||
{
|
||||
@@ -58,6 +62,11 @@ namespace Shadowsocks.Controller
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
if (gfwlistUpdater != null)
|
||||
{
|
||||
gfwlistUpdater.Stop();
|
||||
gfwlistUpdater = null;
|
||||
}
|
||||
if (_listener != null)
|
||||
{
|
||||
_listener.Close();
|
||||
@@ -242,5 +251,73 @@ Connection: Close
|
||||
//}
|
||||
return proxy;
|
||||
}
|
||||
|
||||
private void StartGfwListUpdater()
|
||||
{
|
||||
if (gfwlistUpdater != null)
|
||||
{
|
||||
gfwlistUpdater.Stop();
|
||||
gfwlistUpdater = null;
|
||||
}
|
||||
|
||||
gfwlistUpdater = new GfwListUpdater();
|
||||
gfwlistUpdater.GfwListChanged += gfwlistUpdater_GfwListChanged;
|
||||
IPEndPoint localEndPoint = (IPEndPoint)_listener.LocalEndPoint;
|
||||
gfwlistUpdater.proxy = new WebProxy(localEndPoint.Address.ToString(), 8123);
|
||||
gfwlistUpdater.useSystemProxy = false;
|
||||
/* Delay 30 seconds, wait proxy start up. */
|
||||
gfwlistUpdater.ScheduleUpdateTime(30);
|
||||
gfwlistUpdater.Start();
|
||||
|
||||
}
|
||||
|
||||
private void gfwlistUpdater_GfwListChanged(object sender, GfwListUpdater.GfwListChangedArgs e)
|
||||
{
|
||||
if (e.GfwList == null || e.GfwList.Length == 0) return;
|
||||
string pacfile = TouchPACFile();
|
||||
string pacContent = File.ReadAllText(pacfile);
|
||||
string oldDomains;
|
||||
if (ClearPacContent(ref pacContent, out oldDomains))
|
||||
{
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.AppendLine("{");
|
||||
for (int i = 0; i < e.GfwList.Length; i++)
|
||||
{
|
||||
if (i == e.GfwList.Length - 1)
|
||||
sb.AppendFormat("\t\"{0}\": {1}\r\n", e.GfwList[i], 1);
|
||||
else
|
||||
sb.AppendFormat("\t\"{0}\": {1},\r\n", e.GfwList[i], 1);
|
||||
}
|
||||
sb.Append("}");
|
||||
string newDomains = sb.ToString();
|
||||
if (!string.Equals(oldDomains, newDomains))
|
||||
{
|
||||
pacContent = pacContent.Replace("__LAST_MODIFIED__", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||
pacContent = pacContent.Replace("__DOMAINS__", newDomains);
|
||||
File.WriteAllText(pacfile, pacContent);
|
||||
Console.WriteLine("gfwlist updated - " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("Broken pac file.");
|
||||
}
|
||||
}
|
||||
|
||||
private bool ClearPacContent(ref string pacContent, out string oldDomains)
|
||||
{
|
||||
Regex regex = new Regex("(/\\*.*?\\*/\\s*)?var\\s+domains\\s*=\\s*(\\{(\\s*\"[^\"]*\"\\s*:\\s*\\d+\\s*,)*\\s*(\\s*\"[^\"]*\"\\s*:\\s*\\d+\\s*)\\})", RegexOptions.Singleline);
|
||||
Match m = regex.Match(pacContent);
|
||||
if (m.Success)
|
||||
{
|
||||
oldDomains = m.Result("$2");
|
||||
pacContent = regex.Replace(pacContent, "/* Last Modified: __LAST_MODIFIED__ */\r\nvar domains = __DOMAINS__");
|
||||
return true;
|
||||
}
|
||||
oldDomains = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user