auto update gfwlist, see https://github.com/shadowsocks/shadowsocks-csharp/issues/115

2015-01-08 00:54:12 +08:00
parent a984e90727
commit a2f3116ca8
6 changed files with 399 additions and 24 deletions
@@ -0,0 +1,283 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Net;
+using System.IO;
+using System.IO.Compression;
+using System.Security.Cryptography;
+using Shadowsocks.Model;
+using Shadowsocks.Properties;
+
+namespace Shadowsocks.Controller
+{
+    public class GfwListUpdater
+    {
+        private const string GFWLIST_URL = "https://autoproxy-gfwlist.googlecode.com/svn/trunk/gfwlist.txt";
+
+        private const int EXPIRE_HOURS = 6;
+
+        public IWebProxy proxy = null;
+
+        public bool useSystemProxy = true;
+
+        public class GfwListChangedArgs : EventArgs
+        {
+            public string[] GfwList { get; set; }
+        }
+
+        public event EventHandler<GfwListChangedArgs> GfwListChanged; 
+
+        private bool running = false;
+        private bool closed = false;
+        private int jobId = 0;
+        DateTime lastUpdateTimeUtc;
+        string lastUpdateMd5;
+
+        private object locker = new object();
+
+        public GfwListUpdater()
+        {
+        }
+
+        ~GfwListUpdater()
+        {
+            Stop();
+        }
+
+        public void Start()
+        {
+            lock (locker)
+            {
+                if (running)
+                    return;
+                running = true;
+                closed = false;
+                jobId++;
+                new Thread(new ParameterizedThreadStart(UpdateJob)).Start(jobId);
+            }
+        }
+
+        public void Stop()
+        {
+            lock(locker)
+            {
+                closed = true;
+                running = false;
+                jobId++;
+            }
+        }
+
+        public void ScheduleUpdateTime(int delaySeconds)
+        {
+            lock(locker)
+            {
+                lastUpdateTimeUtc = DateTime.UtcNow.AddHours(-1 * EXPIRE_HOURS).AddSeconds(delaySeconds);
+            }
+        }
+
+        private string DownloadGfwListFile()
+        {
+            try
+            {
+                WebClient http = new WebClient();
+                http.Proxy = useSystemProxy ? WebRequest.GetSystemWebProxy() : proxy;
+                return http.DownloadString(new Uri(GFWLIST_URL));
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex.Message);
+            }
+            return null;
+        }
+
+        private bool IsExpire()
+        {
+            lock (locker)
+            {
+                TimeSpan ts = DateTime.UtcNow - lastUpdateTimeUtc;
+                bool expire = ((int)ts.TotalHours) >= EXPIRE_HOURS;
+                if (expire)
+                    lastUpdateTimeUtc = DateTime.UtcNow;
+                return expire;
+            }
+        }
+
+        private bool IsJobStop(int currentJobId)
+        {
+            lock (locker)
+            {
+                if (!running || closed || currentJobId != this.jobId)
+                    return true;
+            }
+            return false;
+        }
+
+        private bool IsGfwListChanged(string content)
+        {
+            byte[] inputBytes = Encoding.UTF8.GetBytes(content);
+            byte[] md5Bytes = MD5.Create().ComputeHash(inputBytes);
+            string md5 = "";
+            for (int i = 0; i < md5Bytes.Length; i++)
+                md5 += md5Bytes[i].ToString("x").PadLeft(2, '0');
+            if (md5 == lastUpdateMd5)
+                return false;
+            lastUpdateMd5 = md5;
+            return true;
+        }
+
+        private void ParseGfwList(string response)
+        {
+            if (!IsGfwListChanged(response))
+                return;
+            if (GfwListChanged != null)
+            {
+                Parser parser = new Parser(response);
+                GfwListChangedArgs args = new GfwListChangedArgs {
+                    GfwList = parser.GetReducedDomains()
+                };
+                GfwListChanged(this, args);
+            }
+        }
+
+        private void UpdateJob(object state)
+        {
+            int currentJobId = (int)state;
+            int retryTimes = 3;
+            while (!IsJobStop(currentJobId))
+            {
+                if (IsExpire())
+                {
+                    string response = DownloadGfwListFile();
+                    if (response != null)
+                    {
+                        ParseGfwList(response);
+                    }
+                    else if (retryTimes > 0)
+                    {
+                        ScheduleUpdateTime(30); /*Delay 30 seconds to retry*/
+                        retryTimes--;
+                    }
+                    else
+                    {
+                        retryTimes = 3; /* reset retry times, and wait next update time. */
+                    }
+                }
+
+                Thread.Sleep(1000);
+            }
+        }
+
+        class Parser
+        {
+            public string Content { get; private set; }
+
+            public Parser(string response)
+            {
+                byte[] bytes = Convert.FromBase64String(response);
+                this.Content = Encoding.ASCII.GetString(bytes);
+            }
+
+            public string[] GetLines()
+            {
+                return Content.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); 
+            }
+            
+            /* refer https://github.com/clowwindy/gfwlist2pac/blob/master/gfwlist2pac/main.py */
+            public string[] GetDomains()
+            {
+                string[] lines = GetLines();
+                List<string> domains = new List<string>(lines.Length);
+                for(int i =0;i < lines.Length;i++)
+                {
+                    string line = lines[i];
+                    if (line.IndexOf(".*") >= 0)
+                        continue;
+                    else if (line.IndexOf("*") >= 0)
+                        line = line.Replace("*", "/");
+                    if (line.StartsWith("||"))
+                        line = line.Substring(2);
+                    else if (line.StartsWith("|"))
+                        line = line.Substring(1);
+                    else if (line.StartsWith("."))
+                        line = line.Substring(1);
+                    if (line.StartsWith("!"))
+                        continue;
+                    else if (line.StartsWith("["))
+                        continue;
+                    else if (line.StartsWith("@"))
+                        continue; /*ignore white list*/
+                    domains.Add(line);
+                }
+                return domains.ToArray();
+            }
+
+            /* refer https://github.com/clowwindy/gfwlist2pac/blob/master/gfwlist2pac/main.py */
+            public string[] GetReducedDomains()
+            {
+                string[] domains = GetDomains();
+                List<string> new_domains = new List<string>(domains.Length);
+                IDictionary<string, string> tld_dic = GetTldDictionary();
+
+                foreach(string domain in domains)
+                {
+                    string last_root_domain = null;
+                    int pos;
+                    pos = domain.LastIndexOf('.');
+                    last_root_domain = domain.Substring(pos + 1);
+                    if (!tld_dic.ContainsKey(last_root_domain))
+                        continue;
+                    while(pos > 0)
+                    {
+                        pos = domain.LastIndexOf('.', pos - 1);
+                        last_root_domain = domain.Substring(pos + 1);
+                        if (tld_dic.ContainsKey(last_root_domain))
+                            continue;
+                        else
+                            break;
+                    }
+                    if (last_root_domain != null)
+                        new_domains.Add(last_root_domain);
+                }
+
+                return new_domains.ToArray();
+            }
+
+            private string[] GetTlds()
+            {
+                string[] tlds = null;
+                byte[] pacGZ = Resources.tld_txt;
+                byte[] buffer = new byte[1024]; 
+                int n;
+                using(MemoryStream sb = new MemoryStream())
+                {
+                    using (GZipStream input = new GZipStream(new MemoryStream(pacGZ),
+                        CompressionMode.Decompress, false))
+                    {
+                        while ((n = input.Read(buffer, 0, buffer.Length)) > 0)
+                        {
+                            sb.Write(buffer, 0, n);
+                        }
+                    }
+                    tlds = System.Text.Encoding.UTF8.GetString(sb.ToArray())
+                        .Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
+                }
+                return tlds;
+            }
+
+            private IDictionary<string, string> GetTldDictionary()
+            {
+                string[] tlds = GetTlds();
+                IDictionary<string, string> dic = new Dictionary<string, string>(tlds.Length);
+                foreach (string tld in tlds)
+                {
+                    if (!dic.ContainsKey(tld))
+                        dic.Add(tld, tld);
+                }
+                return dic;
+            }
+
+        }
+
+    }
+}
@@ -8,6 +8,7 @@ using System.IO.Compression;
 using System.Net;
 using System.Net.Sockets;
 using System.Text;
+using System.Text.RegularExpressions;

 namespace Shadowsocks.Controller
 {
@@ -20,6 +21,8 @@ namespace Shadowsocks.Controller
        Socket _listener;
        FileSystemWatcher watcher;

+        GfwListUpdater gfwlistUpdater;
+
        public event EventHandler PACFileChanged;

        public void Start(Configuration configuration)
@@ -48,6 +51,7 @@ namespace Shadowsocks.Controller
                    _listener);

                WatchPacFile();
+                StartGfwListUpdater();
            }
            catch (SocketException)
            {
@@ -58,6 +62,11 @@ namespace Shadowsocks.Controller

        public void Stop()
        {
+            if (gfwlistUpdater != null)
+            {
+                gfwlistUpdater.Stop();
+                gfwlistUpdater = null;
+            }
            if (_listener != null)
            {
                _listener.Close();
@@ -242,5 +251,73 @@ Connection: Close
            //}
            return proxy;
        }
+
+        private void StartGfwListUpdater()
+        {
+            if (gfwlistUpdater != null)
+            {
+                gfwlistUpdater.Stop();
+                gfwlistUpdater = null;
+            }
+
+            gfwlistUpdater = new GfwListUpdater();
+            gfwlistUpdater.GfwListChanged += gfwlistUpdater_GfwListChanged;
+            IPEndPoint localEndPoint = (IPEndPoint)_listener.LocalEndPoint;
+            gfwlistUpdater.proxy = new WebProxy(localEndPoint.Address.ToString(), 8123);
+            gfwlistUpdater.useSystemProxy = false;
+            /* Delay 30 seconds, wait proxy start up. */
+            gfwlistUpdater.ScheduleUpdateTime(30);
+            gfwlistUpdater.Start();
+
+        }
+
+        private void gfwlistUpdater_GfwListChanged(object sender, GfwListUpdater.GfwListChangedArgs e)
+        {
+            if (e.GfwList == null || e.GfwList.Length == 0) return;
+            string pacfile = TouchPACFile();
+            string pacContent = File.ReadAllText(pacfile);
+            string oldDomains;
+            if (ClearPacContent(ref pacContent, out oldDomains))
+            {
+                StringBuilder sb = new StringBuilder();
+                sb.AppendLine("{");
+                for (int i = 0; i < e.GfwList.Length; i++)
+                {
+                    if (i == e.GfwList.Length - 1)
+                        sb.AppendFormat("\t\"{0}\": {1}\r\n", e.GfwList[i], 1);
+                    else
+                        sb.AppendFormat("\t\"{0}\": {1},\r\n", e.GfwList[i], 1);
+                }
+                sb.Append("}");
+                string newDomains = sb.ToString();
+                if (!string.Equals(oldDomains, newDomains))
+                {
+                    pacContent = pacContent.Replace("__LAST_MODIFIED__", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
+                    pacContent = pacContent.Replace("__DOMAINS__", newDomains);
+                    File.WriteAllText(pacfile, pacContent);
+                    Console.WriteLine("gfwlist updated - " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
+                }
+            }
+            else
+            {
+                Console.WriteLine("Broken pac file.");
+            }
+        }
+
+        private bool ClearPacContent(ref string pacContent, out string oldDomains)
+        {
+            Regex regex = new Regex("(/\\*.*?\\*/\\s*)?var\\s+domains\\s*=\\s*(\\{(\\s*\"[^\"]*\"\\s*:\\s*\\d+\\s*,)*\\s*(\\s*\"[^\"]*\"\\s*:\\s*\\d+\\s*)\\})", RegexOptions.Singleline);
+            Match m = regex.Match(pacContent);
+            if (m.Success)
+            {
+                oldDomains = m.Result("$2");
+                pacContent = regex.Replace(pacContent, "/* Last Modified: __LAST_MODIFIED__ */\r\nvar domains = __DOMAINS__");
+                return true;
+            }
+            oldDomains = null;
+            return false;
+        }
+
+
    }
 }
@@ -1,10 +1,10 @@
 //------------------------------------------------------------------------------
 // <auto-generated>
-//     This code was generated by a tool.
-//     Runtime Version:4.0.30319.18444
+//     此代码由工具生成。
+//     运行时版本:4.0.30319.34014
 //
-//     Changes to this file may cause incorrect behavior and will be lost if
-//     the code is regenerated.
+//     对此文件的更改可能会导致不正确的行为，并且如果
+//     重新生成代码，这些更改将会丢失。
 // </auto-generated>
 //------------------------------------------------------------------------------

@@ -13,12 +13,12 @@ namespace Shadowsocks.Properties {
    
    
    /// <summary>
-    ///   A strongly-typed resource class, for looking up localized strings, etc.
+    ///   一个强类型的资源类，用于查找本地化的字符串等。
    /// </summary>
-    // This class was auto-generated by the StronglyTypedResourceBuilder
-    // class via a tool like ResGen or Visual Studio.
-    // To add or remove a member, edit your .ResX file then rerun ResGen
-    // with the /str option, or rebuild your VS project.
+    // 此类是由 StronglyTypedResourceBuilder
+    // 类通过类似于 ResGen 或 Visual Studio 的工具自动生成的。
+    // 若要添加或移除成员，请编辑 .ResX 文件，然后重新运行 ResGen
+    // (以 /str 作为命令选项)，或重新生成 VS 项目。
    [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")]
    [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
    [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
@@ -33,7 +33,7 @@ namespace Shadowsocks.Properties {
        }
        
        /// <summary>
-        ///   Returns the cached ResourceManager instance used by this class.
+        ///   返回此类使用的缓存的 ResourceManager 实例。
        /// </summary>
        [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
        internal static global::System.Resources.ResourceManager ResourceManager {
@@ -47,8 +47,8 @@ namespace Shadowsocks.Properties {
        }
        
        /// <summary>
-        ///   Overrides the current thread's CurrentUICulture property for all
-        ///   resource lookups using this strongly typed resource class.
+        ///   使用此强类型资源类，为所有资源查找
+        ///   重写当前线程的 CurrentUICulture 属性。
        /// </summary>
        [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
        internal static global::System.Globalization.CultureInfo Culture {
@@ -61,14 +61,14 @@ namespace Shadowsocks.Properties {
        }
        
        /// <summary>
-        ///   Looks up a localized string similar to Shadowsocks=Shadowsocks
+        ///   查找类似 Shadowsocks=Shadowsocks
        ///Enable=启用代理
        ///Mode=代理模式
        ///PAC=PAC 模式
        ///Global=全局模式
        ///Servers=服务器选择
        ///Edit Servers...=编辑服务器...
-        ///Start on Boot=自动启动
+        ///Start on Boot=开机启动
        ///Share over LAN=在局域网共享代理
        ///Edit PAC File...=编辑 PAC 文件...
        ///Show QRCode...=显示二维码...
@@ -91,7 +91,7 @@ namespace Shadowsocks.Properties {
        ///QRCode=二维码
        ///Shadowsocks Error: {0}=Shadowsocks 错误: {0}
        ///Port already in use=端口已被占用
-        ///Il [rest of string was truncated]&quot;;.
+        ///Il [字符串的其余部分被截断]&quot;; 的本地化字符串。
        /// </summary>
        internal static string cn {
            get {
@@ -100,7 +100,7 @@ namespace Shadowsocks.Properties {
        }
        
        /// <summary>
-        ///   Looks up a localized resource of type System.Byte[].
+        ///   查找 System.Byte[] 类型的本地化资源。
        /// </summary>
        internal static byte[] libsscrypto_dll {
            get {
@@ -110,7 +110,7 @@ namespace Shadowsocks.Properties {
        }
        
        /// <summary>
-        ///   Looks up a localized string similar to proxyAddress = &quot;__POLIPO_BIND_IP__&quot;
+        ///   查找类似 proxyAddress = &quot;__POLIPO_BIND_IP__&quot;
        ///
        ///socksParentProxy = &quot;127.0.0.1:__SOCKS_PORT__&quot;
        ///socksProxyType = socks5
@@ -118,7 +118,7 @@ namespace Shadowsocks.Properties {
        ///localDocumentRoot = &quot;&quot;
        ///
        ///allowedPorts = 1-65535
-        ///tunnelAllowedPorts = 1-65535.
+        ///tunnelAllowedPorts = 1-65535 的本地化字符串。
        /// </summary>
        internal static string polipo_config {
            get {
@@ -127,7 +127,7 @@ namespace Shadowsocks.Properties {
        }
        
        /// <summary>
-        ///   Looks up a localized resource of type System.Byte[].
+        ///   查找 System.Byte[] 类型的本地化资源。
        /// </summary>
        internal static byte[] polipo_exe {
            get {
@@ -137,7 +137,7 @@ namespace Shadowsocks.Properties {
        }
        
        /// <summary>
-        ///   Looks up a localized resource of type System.Byte[].
+        ///   查找 System.Byte[] 类型的本地化资源。
        /// </summary>
        internal static byte[] proxy_pac_txt {
            get {
@@ -147,7 +147,7 @@ namespace Shadowsocks.Properties {
        }
        
        /// <summary>
-        ///   Looks up a localized resource of type System.Drawing.Bitmap.
+        ///   查找 System.Drawing.Bitmap 类型的本地化资源。
        /// </summary>
        internal static System.Drawing.Bitmap ss16 {
            get {
@@ -157,7 +157,7 @@ namespace Shadowsocks.Properties {
        }
        
        /// <summary>
-        ///   Looks up a localized resource of type System.Drawing.Bitmap.
+        ///   查找 System.Drawing.Bitmap 类型的本地化资源。
        /// </summary>
        internal static System.Drawing.Bitmap ss20 {
            get {
@@ -167,7 +167,7 @@ namespace Shadowsocks.Properties {
        }
        
        /// <summary>
-        ///   Looks up a localized resource of type System.Drawing.Bitmap.
+        ///   查找 System.Drawing.Bitmap 类型的本地化资源。
        /// </summary>
        internal static System.Drawing.Bitmap ss24 {
            get {
@@ -177,7 +177,7 @@ namespace Shadowsocks.Properties {
        }
        
        /// <summary>
-        ///   Looks up a localized resource of type System.Drawing.Bitmap.
+        ///   查找 System.Drawing.Bitmap 类型的本地化资源。
        /// </summary>
        internal static System.Drawing.Bitmap ssw128 {
            get {
@@ -185,5 +185,15 @@ namespace Shadowsocks.Properties {
                return ((System.Drawing.Bitmap)(obj));
            }
        }
+        
+        /// <summary>
+        ///   查找 System.Byte[] 类型的本地化资源。
+        /// </summary>
+        internal static byte[] tld_txt {
+            get {
+                object obj = ResourceManager.GetObject("tld_txt", resourceCulture);
+                return ((byte[])(obj));
+            }
+        }
    }
 }
@@ -145,4 +145,7 @@
  <data name="ssw128" type="System.Resources.ResXFileRef, System.Windows.Forms">
    <value>..\Resources\ssw128.png;System.Drawing.Bitmap, System.Drawing, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
  </data>
+  <data name="tld_txt" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Data\tld.txt.gz;System.Byte[], mscorlib, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </data>
 </root>
@@ -86,6 +86,7 @@
    <Compile Include="3rd\zxing\Version.cs" />
    <Compile Include="Controller\AutoStartup.cs" />
    <Compile Include="Controller\FileManager.cs" />
+    <Compile Include="Controller\GfwListUpdater.cs" />
    <Compile Include="Controller\I18N.cs" />
    <Compile Include="Controller\Logging.cs" />
    <Compile Include="Controller\UpdateChecker.cs" />
@@ -147,6 +148,7 @@
    <None Include="Data\proxy.pac.txt.gz" />
  </ItemGroup>
  <ItemGroup>
+    <None Include="Data\tld.txt.gz" />
    <None Include="Resources\ss20.png" />
    <None Include="Resources\ss16.png" />
    <None Include="Resources\ss24.png" />