Alawn/shadowsocks-windows
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

move loading and filtering jobs to statistics service

Browse Source
This commit is contained in:
icylogic
2015-09-14 20:57:17 +08:00
parent e45dd8b695
commit 107bc99282
4 changed files with 137 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
shadowsocks-csharp/Controller/Service/AvailabilityStatistics.cs
+118 -8
View File
@@ -1,32 +1,36 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using SimpleJson;
using System.Net.Http;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using Shadowsocks.Model;
using Shadowsocks.Util;
using Timer = System.Threading.Timer;
namespace Shadowsocks.Controller
{
using DataUnit = KeyValuePair<string, string>;
using DataList = List<KeyValuePair<string, string>>;
internal class AvailabilityStatistics
using RawStatistics = Dictionary<string, List<AvailabilityStatistics.RawStatisticsData>>;
using Statistics = Dictionary<string, List<AvailabilityStatistics.StatisticsData>>;
public class AvailabilityStatistics
{
public static readonly string DateTimePattern = "yyyy-MM-dd HH:mm:ss";
private const string StatisticsFilesName = "shadowsocks.availability.csv";
private const string Delimiter = ",";
private const int Timeout = 500;
private const int DelayBeforeStart = 1000;
public RawStatistics rawStatistics { get; private set; }
public RawStatistics filteredStatistics { get; private set; }
private int _repeat => _config.RepeatTimesNum;
private const int RetryInterval = 2*60*1000; //retry 2 minutes after failed
private int _interval => (int) TimeSpan.FromMinutes(_config.DataCollectionMinutes).TotalMilliseconds;
private Timer _timer;
private State _state;
@@ -57,7 +61,7 @@ namespace Shadowsocks.Controller
if (_timer?.Change(DelayBeforeStart, _interval) == null)
{
_state = new State();
_timer = new Timer(Evaluate, _state, DelayBeforeStart, _interval);
_timer = new Timer(run, _state, DelayBeforeStart, _interval);
}
}
else
@@ -82,7 +86,7 @@ namespace Shadowsocks.Controller
var ret = new DataList
{
new DataUnit(State.Geolocation, State.Unknown),
new DataUnit(State.ISP, State.Unknown),
new DataUnit(State.ISP, State.Unknown)
};
string jsonString;
try
@@ -95,7 +99,7 @@ namespace Shadowsocks.Controller
return ret;
}
dynamic obj;
if (!global::SimpleJson.SimpleJson.TryDeserializeObject(jsonString, out obj)) return ret;
if (!SimpleJson.SimpleJson.TryDeserializeObject(jsonString, out obj)) return ret;
string country = obj["country"];
string city = obj["city"];
string isp = obj["isp"];
@@ -140,7 +144,14 @@ namespace Shadowsocks.Controller
return ret;
}
private async void Evaluate(object obj)
private void run(object obj)
{
LoadRawStatistics();
FilterRawStatistics();
evaluate();
}
private async void evaluate()
{
var geolocationAndIsp = GetGeolocationAndIsp();
foreach (var dataLists in await TaskEx.WhenAll(_servers.Select(ICMPTest)))
@@ -184,6 +195,86 @@ namespace Shadowsocks.Controller
_servers = config.configs;
}
private void FilterRawStatistics()
{
if (filteredStatistics == null)
{
filteredStatistics = new RawStatistics();
}
foreach (IEnumerable<RawStatisticsData> rawData in rawStatistics.Values)
{
var filteredData = rawData;
if (_config.ByIsp)
{
var current = GetGeolocationAndIsp().Result;
filteredData =
filteredData.Where(
data =>
data.Geolocation == current[0].Value ||
data.Geolocation == State.Unknown);
filteredData =
filteredData.Where(
data => data.ISP == current[1].Value || data.ISP == State.Unknown);
if (filteredData.LongCount() == 0) return;
}
if (_config.ByHourOfDay)
{
var currentHour = DateTime.Now.Hour;
filteredData = filteredData.Where(data =>
{
DateTime dateTime;
DateTime.TryParseExact(data.Timestamp, DateTimePattern, null,
DateTimeStyles.None, out dateTime);
var result = dateTime.Hour.Equals(currentHour);
return result;
});
if (filteredData.LongCount() == 0) return;
}
var dataList = filteredData as List<RawStatisticsData> ?? filteredData.ToList();
var serverName = dataList[0].ServerName;
filteredStatistics[serverName] = dataList;
}
}
private void LoadRawStatistics()
{
try
{
var path = AvailabilityStatisticsFile;
Logging.Debug($"loading statistics from {path}");
if (!File.Exists(path))
{
Console.WriteLine($"statistics file does not exist, try to reload {RetryInterval/60/1000} minutes later");
_timer.Change(RetryInterval, _interval);
return;
}
rawStatistics = (from l in File.ReadAllLines(path)
.Skip(1)
let strings = l.Split(new[] {","}, StringSplitOptions.RemoveEmptyEntries)
let rawData = new RawStatisticsData
{
Timestamp = strings[0],
ServerName = strings[1],
ICMPStatus = strings[2],
RoundtripTime = int.Parse(strings[3]),
Geolocation = 5 > strings.Length ?
null
: strings[4],
ISP = 6 > strings.Length ? null : strings[5]
}
group rawData by rawData.ServerName into server
select new
{
ServerName = server.Key,
data = server.ToList()
}).ToDictionary(server => server.ServerName, server=> server.data);
}
catch (Exception e)
{
Logging.LogUsefulException(e);
}
}
public class State
{
public DataList dataList = new DataList();
@@ -191,5 +282,24 @@ namespace Shadowsocks.Controller
public const string ISP = "ISP";
public const string Unknown = "Unknown";
}
public class RawStatisticsData
{
public string Timestamp;
public string ServerName;
public string ICMPStatus;
public int RoundtripTime;
public string Geolocation;
public string ISP ;
}
public class StatisticsData
{
public float PackageLoss;
public int AverageResponse;
public int MinResponse;
public int MaxResponse;
}
}
}
shadowsocks-csharp/Controller/ShadowsocksController.cs
+6 -6
View File
@@ -25,7 +25,7 @@ namespace Shadowsocks.Controller
private StrategyManager _strategyManager;
private PolipoRunner polipoRunner;
private GFWListUpdater gfwListUpdater;
private AvailabilityStatistics _availabilityStatics;
public AvailabilityStatistics availabilityStatistics { get; private set; }
public StatisticsStrategyConfiguration StatisticsConfiguration { get; private set; }
private bool stopped = false;
@@ -260,8 +260,8 @@ namespace Shadowsocks.Controller
public void UpdateStatisticsConfiguration(bool enabled)
{
if (_availabilityStatics == null) return;
_availabilityStatics.UpdateConfiguration(_config, StatisticsConfiguration);
if (availabilityStatistics == null) return;
availabilityStatistics.UpdateConfiguration(_config, StatisticsConfiguration);
_config.availabilityStatistics = enabled;
SaveConfig(_config);
}
@@ -311,11 +311,11 @@ namespace Shadowsocks.Controller
gfwListUpdater.Error += pacServer_PACUpdateError;
}
if (_availabilityStatics == null)
if (availabilityStatistics == null)
{
_availabilityStatics = new AvailabilityStatistics(_config, StatisticsConfiguration);
availabilityStatistics = new AvailabilityStatistics(_config, StatisticsConfiguration);
}
_availabilityStatics.UpdateConfiguration(_config, StatisticsConfiguration);
availabilityStatistics.UpdateConfiguration(_config, StatisticsConfiguration);
if (_listener != null)
{
shadowsocks-csharp/Controller/Strategy/StatisticsStrategy.cs
+11 -91
View File
@@ -4,29 +4,21 @@ using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using Shadowsocks.Model;
using System.IO;
using System.Net.NetworkInformation;
using System.Windows.Forms;
using System.Threading;
using Newtonsoft.Json;
using Shadowsocks.Model;
using Timer = System.Threading.Timer;
namespace Shadowsocks.Controller.Strategy
{
using DataUnit = KeyValuePair<string, string>;
using DataList = List<KeyValuePair<string, string>>;
class StatisticsStrategy : IStrategy
{
private readonly ShadowsocksController _controller;
private Server _currentServer;
private readonly Timer _timer;
private Dictionary<string, List<StatisticsRawData>> _rawStatistics;
private Dictionary<string, List<AvailabilityStatistics.RawStatisticsData>> _filteredStatistics;
private int ChoiceKeptMilliseconds
=> (int) TimeSpan.FromMinutes(_controller.StatisticsConfiguration.ChoiceKeptMinutes).TotalMilliseconds;
private const int RetryInterval = 2*60*1000; //retry 2 minutes after failed
public StatisticsStrategy(ShadowsocksController controller)
{
@@ -47,75 +39,21 @@ namespace Shadowsocks.Controller.Strategy
private void LoadStatistics()
{
try
{
var path = AvailabilityStatistics.AvailabilityStatisticsFile;
Logging.Debug($"loading statistics from {path}");
if (!File.Exists(path))
{
LogWhenEnabled($"statistics file does not exist, try to reload {RetryInterval/60/1000} minutes later");
_timer.Change(RetryInterval, ChoiceKeptMilliseconds);
return;
}
_rawStatistics = (from l in File.ReadAllLines(path)
.Skip(1)
let strings = l.Split(new[] {","}, StringSplitOptions.RemoveEmptyEntries)
let rawData = new StatisticsRawData
{
Timestamp = strings[0],
ServerName = strings[1],
ICMPStatus = strings[2],
RoundtripTime = int.Parse(strings[3]),
Geolocation = 5 > strings.Length ?
null
: strings[4],
ISP = 6 > strings.Length ? null : strings[5]
}
group rawData by rawData.ServerName into server
select new
{
ServerName = server.Key,
data = server.ToList()
}).ToDictionary(server => server.ServerName, server=> server.data);
}
catch (Exception e)
{
Logging.LogUsefulException(e);
}
_filteredStatistics = _controller.availabilityStatistics.rawStatistics ?? _filteredStatistics ?? new Dictionary<string, List<AvailabilityStatistics.RawStatisticsData>>();
}
//return the score by data
//server with highest score will be choosen
private float GetScore(IEnumerable<StatisticsRawData> rawDataList)
private float GetScore(string serverName)
{
var config = _controller.StatisticsConfiguration;
if (config.ByIsp)
{
var current = AvailabilityStatistics.GetGeolocationAndIsp().Result;
rawDataList = rawDataList.Where(data => data.Geolocation == current[0].Value || data.Geolocation == AvailabilityStatistics.State.Unknown);
rawDataList = rawDataList.Where(data => data.ISP == current[1].Value || data.ISP == AvailabilityStatistics.State.Unknown);
if (rawDataList.LongCount() == 0) return 0;
}
if (config.ByHourOfDay)
{
var currentHour = DateTime.Now.Hour;
rawDataList = rawDataList.Where(data =>
{
DateTime dateTime;
DateTime.TryParseExact(data.Timestamp, AvailabilityStatistics.DateTimePattern, null,
DateTimeStyles.None, out dateTime);
var result = dateTime.Hour.Equals(currentHour);
return result;
});
if (rawDataList.LongCount() == 0) return 0;
}
var dataList = rawDataList as IList<StatisticsRawData> ?? rawDataList.ToList();
var serverName = dataList[0]?.ServerName;
List<AvailabilityStatistics.RawStatisticsData> dataList;
if (_filteredStatistics == null || !_filteredStatistics.TryGetValue(serverName, out dataList)) return 0;
var SuccessTimes = (float) dataList.Count(data => data.ICMPStatus.Equals(IPStatus.Success.ToString()));
var TimedOutTimes = (float) dataList.Count(data => data.ICMPStatus.Equals(IPStatus.TimedOut.ToString()));
var statisticsData = new StatisticsData()
var statisticsData = new AvailabilityStatistics.StatisticsData()
{
PackageLoss = TimedOutTimes / (SuccessTimes + TimedOutTimes) * 100,
PackageLoss = TimedOutTimes/(SuccessTimes + TimedOutTimes)*100,
AverageResponse = Convert.ToInt32(dataList.Average(data => data.RoundtripTime)),
MinResponse = dataList.Min(data => data.RoundtripTime),
MaxResponse = dataList.Max(data => data.RoundtripTime)
@@ -134,27 +72,9 @@ namespace Shadowsocks.Controller.Strategy
return score;
}
class StatisticsRawData
{
public string Timestamp;
public string ServerName;
public string ICMPStatus;
public int RoundtripTime;
public string Geolocation;
public string ISP ;
}
public class StatisticsData
{
public float PackageLoss;
public int AverageResponse;
public int MinResponse;
public int MaxResponse;
}
private void ChooseNewServer(List<Server> servers)
{
if (_rawStatistics == null || servers.Count == 0)
if (_filteredStatistics == null || servers.Count == 0)
{
return;
}
@@ -162,11 +82,11 @@ namespace Shadowsocks.Controller.Strategy
{
var bestResult = (from server in servers
let name = server.FriendlyName()
where _rawStatistics.ContainsKey(name)
where _filteredStatistics.ContainsKey(name)
select new
{
server,
score = GetScore(_rawStatistics[name])
score = GetScore(name)
}
).Aggregate((result1, result2) => result1.score > result2.score ? result1 : result2);
shadowsocks-csharp/Model/StatisticsStrategyConfiguration.cs
+2 -2
View File
@@ -62,8 +62,8 @@ namespace Shadowsocks.Model
public StatisticsStrategyConfiguration()
{
var statisticsStrategy = typeof (StatisticsStrategy);
var statisticsData = statisticsStrategy.GetNestedType("StatisticsData");
var availabilityStatisticsType = typeof (AvailabilityStatistics);
var statisticsData = availabilityStatisticsType.GetNestedType("StatisticsData");
var properties = statisticsData.GetFields(BindingFlags.Instance | BindingFlags.Public);
Calculations = properties.ToDictionary(p => p.Name, _ => (float) 0);
}