From e10c681671dfccba6b1cc7acd0e490b87179af9e Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 25 Aug 2013 14:17:05 +0800 Subject: [PATCH] improve sql wall --- .../alibaba/druid/sql/parser/CharTypes.java | 23 ++- .../sql/visitor/SQLEvalVisitorUtils.java | 29 +++- .../visitor/functions/OneParamFunctions.java | 159 ++++++++++++++++++ .../java/com/alibaba/druid/util/HexBin.java | 91 +++++----- .../java/com/alibaba/druid/util/Utils.java | 24 +++ .../druid/wall/spi/WallVisitorUtils.java | 7 +- .../bvt/filter/wall/MySqlWallTest106.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest107.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest108.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest109.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest110.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest111.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest112.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest113.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest114.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest115.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest116.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest117.java | 36 ++++ .../bvt/filter/wall/MySqlWallTest118.java | 36 ++++ 19 files changed, 741 insertions(+), 60 deletions(-) create mode 100644 src/main/java/com/alibaba/druid/sql/visitor/functions/OneParamFunctions.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest106.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest107.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest108.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest109.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest110.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest111.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest112.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest113.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest114.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest115.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest116.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest117.java create mode 100644 src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallTest118.java diff --git a/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java b/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java index 84d601e7b5..522bd9e0a1 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java +++ b/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java @@ -15,6 +15,8 @@ */ package com.alibaba.druid.sql.parser; +import static com.alibaba.druid.sql.parser.LayoutCharacters.EOI; + public class CharTypes { private final static boolean[] hexFlags = new boolean[256]; @@ -53,9 +55,9 @@ public class CharTypes { } public static boolean isFirstIdentifierChar(char c) { - if (c <= firstIdentifierFlags.length) { - return firstIdentifierFlags[c]; - } + if (c <= firstIdentifierFlags.length) { + return firstIdentifierFlags[c]; + } return c != '　' && c != '，'; } @@ -85,12 +87,15 @@ public class CharTypes { private final static boolean[] whitespaceFlags = new boolean[256]; static { - whitespaceFlags[' '] = true; - whitespaceFlags['\n'] = true; - whitespaceFlags['\r'] = true; - whitespaceFlags['\t'] = true; - whitespaceFlags['\f'] = true; - whitespaceFlags['\b'] = true; + for (int i = 0; i <= 32; ++i) { + whitespaceFlags[i] = true; + } + + whitespaceFlags[EOI] = false; + for (int i = 0x7F; i <= 0xA0; ++i) { + whitespaceFlags[i] = true; + } + whitespaceFlags[160] = true; // 特别处理 } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLEvalVisitorUtils.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLEvalVisitorUtils.java index 3dfa190739..8ccf8f0cb4 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLEvalVisitorUtils.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLEvalVisitorUtils.java @@ -26,6 +26,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.BitSet; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -81,6 +82,7 @@ import com.alibaba.druid.sql.visitor.functions.Locate; import com.alibaba.druid.sql.visitor.functions.Lpad; import com.alibaba.druid.sql.visitor.functions.Ltrim; import com.alibaba.druid.sql.visitor.functions.Now; +import com.alibaba.druid.sql.visitor.functions.OneParamFunctions; import com.alibaba.druid.sql.visitor.functions.Reverse; import com.alibaba.druid.sql.visitor.functions.Right; import com.alibaba.druid.sql.visitor.functions.Substring; @@ -215,6 +217,11 @@ public class SQLEvalVisitorUtils { functions.put("least", Least.instance); functions.put("isnull", Isnull.instance); functions.put("if", If.instance); + + functions.put("md5", OneParamFunctions.instance); + functions.put("bit_count", OneParamFunctions.instance); + functions.put("soundex", OneParamFunctions.instance); + functions.put("space", OneParamFunctions.instance); } public static boolean visit(SQLEvalVisitor visitor, SQLMethodInvokeExpr x) { @@ -568,8 +575,28 @@ public class SQLEvalVisitorUtils { } public static boolean visit(SQLEvalVisitor visitor, MySqlBinaryExpr x) { + String text = x.getValue(); + BitSet bits = new BitSet(text.length()); + for (int i = 0; i < text.length(); ++i) { + char ch = text.charAt(i); + if (ch == '1') { + bits.set(i); + } + } - return true; + long[] longValues = bits.toLongArray(); + + Object val; + + if (longValues.length == 1) { + val = longValues[0]; + } else { + val = new BigInteger(bits.toByteArray()); + } + + x.putAttribute(EVAL_VALUE, val); + + return false; } public static boolean visit(SQLEvalVisitor visitor, SQLBetweenExpr x) { diff --git a/src/main/java/com/alibaba/druid/sql/visitor/functions/OneParamFunctions.java b/src/main/java/com/alibaba/druid/sql/visitor/functions/OneParamFunctions.java new file mode 100644 index 0000000000..01e8898a31 --- /dev/null +++ b/src/main/java/com/alibaba/druid/sql/visitor/functions/OneParamFunctions.java @@ -0,0 +1,159 @@ +/* + * Copyright 1999-2011 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.sql.visitor.functions; + +import static com.alibaba.druid.sql.visitor.SQLEvalVisitor.EVAL_VALUE; +import static com.alibaba.druid.sql.visitor.SQLEvalVisitor.EVAL_VALUE_NULL; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; +import com.alibaba.druid.sql.visitor.SQLEvalVisitor; +import com.alibaba.druid.sql.visitor.SQLEvalVisitorUtils; +import com.alibaba.druid.util.Utils; + +public class OneParamFunctions implements Function { + + public final static OneParamFunctions instance = new OneParamFunctions(); + + public Object eval(SQLEvalVisitor visitor, SQLMethodInvokeExpr x) { + if (x.getParameters().size() == 0) { + return SQLEvalVisitor.EVAL_ERROR; + } + + SQLExpr param = x.getParameters().get(0); + param.accept(visitor); + + Object paramValue = param.getAttributes().get(EVAL_VALUE); + if (paramValue == null) { + return SQLEvalVisitor.EVAL_ERROR; + } + + if (paramValue == EVAL_VALUE_NULL) { + return EVAL_VALUE_NULL; + } + + String method = x.getMethodName(); + if ("md5".equalsIgnoreCase(method)) { + String text = paramValue.toString(); + return Utils.md5(text); + } + + if ("bit_count".equalsIgnoreCase(method)) { + if (paramValue instanceof BigInteger) { + return ((BigInteger) paramValue).bitCount(); + } + + if (paramValue instanceof BigDecimal) { + BigDecimal decimal = (BigDecimal) paramValue; + BigInteger bigInt = decimal.setScale(0, BigDecimal.ROUND_HALF_UP).toBigInteger(); + return bigInt.bitCount(); + } + Long val = SQLEvalVisitorUtils.castToLong(paramValue); + return Long.bitCount(val); + } + + if ("soundex".equalsIgnoreCase(method)) { + String text = paramValue.toString(); + return soundex(text); + } + + if ("space".equalsIgnoreCase(method)) { + int intVal = SQLEvalVisitorUtils.castToInteger(paramValue); + char[] chars = new char[intVal]; + for (int i = 0; i < chars.length; ++i) { + chars[i] = ' '; + } + return new String(chars); + } + + throw new UnsupportedOperationException(method); + } + + public static String soundex(String str) { + if (str == null) { + return null; + } + str = clean(str); + if (str.length() == 0) { + return str; + } + char out[] = {'0', '0', '0', '0'}; + char last, mapped; + int incount = 1, count = 1; + out[0] = str.charAt(0); + // getMappingCode() throws IllegalArgumentException + last = getMappingCode(str, 0); + while ((incount < str.length()) && (count < out.length)) { + mapped = getMappingCode(str, incount++); + if (mapped != 0) { + if ((mapped != '0') && (mapped != last)) { + out[count++] = mapped; + } + last = mapped; + } + } + return new String(out); + } + + static String clean(String str) { + if (str == null || str.length() == 0) { + return str; + } + int len = str.length(); + char[] chars = new char[len]; + int count = 0; + for (int i = 0; i < len; i++) { + if (Character.isLetter(str.charAt(i))) { + chars[count++] = str.charAt(i); + } + } + if (count == len) { + return str.toUpperCase(java.util.Locale.ENGLISH); + } + return new String(chars, 0, count).toUpperCase(java.util.Locale.ENGLISH); + } + + private static char getMappingCode(String str, int index) { + // map() throws IllegalArgumentException + char mappedChar = map(str.charAt(index)); + // HW rule check + if (index > 1 && mappedChar != '0') { + char hwChar = str.charAt(index - 1); + if ('H' == hwChar || 'W' == hwChar) { + char preHWChar = str.charAt(index - 2); + char firstCode = map(preHWChar); + if (firstCode == mappedChar || 'H' == preHWChar || 'W' == preHWChar) { + return 0; + } + } + } + return mappedChar; + } + + private static char map(char ch) { + String soundexMapping = "01230120022455012623010202"; + int index = ch - 'A'; + if (index < 0 || index >= soundexMapping.length()) { + throw new IllegalArgumentException("The character is not mapped: " + ch); + } + return soundexMapping.charAt(index); + } + + +} diff --git a/src/main/java/com/alibaba/druid/util/HexBin.java b/src/main/java/com/alibaba/druid/util/HexBin.java index 809e4fb4d9..9c7fa8e78f 100644 --- a/src/main/java/com/alibaba/druid/util/HexBin.java +++ b/src/main/java/com/alibaba/druid/util/HexBin.java @@ -15,75 +15,68 @@ */ package com.alibaba.druid.util; - /** - * format validation - * - * This class encodes/decodes hexadecimal data - * - * @xerces.internal + * format validation This class encodes/decodes hexadecimal data * + * @xerces.internal * @author Jeffrey Rodriguez * @version $Id: HexBin.java,v 1.4 2007/07/19 04:38:32 ofung Exp $ */ -public final class HexBin { - static private final int BASELENGTH = 128; - static private final int LOOKUPLENGTH = 16; - static final private byte [] hexNumberTable = new byte[BASELENGTH]; - static final private char [] lookUpHexAlphabet = new char[LOOKUPLENGTH]; +public final class HexBin { + static private final int BASELENGTH = 128; + static private final int LOOKUPLENGTH = 16; + static final private byte[] hexNumberTable = new byte[BASELENGTH]; + static final private char[] upperChars = new char[LOOKUPLENGTH]; + static final private char[] lowerChars = new char[LOOKUPLENGTH]; static { - for (int i = 0; i < BASELENGTH; i++ ) { + for (int i = 0; i < BASELENGTH; i++) { hexNumberTable[i] = -1; } - for ( int i = '9'; i >= '0'; i--) { - hexNumberTable[i] = (byte) (i-'0'); + for (int i = '9'; i >= '0'; i--) { + hexNumberTable[i] = (byte) (i - '0'); } - for ( int i = 'F'; i>= 'A'; i--) { - hexNumberTable[i] = (byte) ( i-'A' + 10 ); + for (int i = 'F'; i >= 'A'; i--) { + hexNumberTable[i] = (byte) (i - 'A' + 10); } - for ( int i = 'f'; i>= 'a'; i--) { - hexNumberTable[i] = (byte) ( i-'a' + 10 ); + for (int i = 'f'; i >= 'a'; i--) { + hexNumberTable[i] = (byte) (i - 'a' + 10); } - for(int i = 0; i<10; i++ ) { - lookUpHexAlphabet[i] = (char)('0'+i); + for (int i = 0; i < 10; i++) { + upperChars[i] = (char) ('0' + i); + lowerChars[i] = (char) ('0' + i); } - for(int i = 10; i<=15; i++ ) { - lookUpHexAlphabet[i] = (char)('A'+i -10); + for (int i = 10; i <= 15; i++) { + upperChars[i] = (char) ('A' + i - 10); + lowerChars[i] = (char) ('a' + i - 10); } } + + public static String encode(byte[] bytes) { + return encode(bytes, true); + } - /** - * Encode a byte array to hex string - * - * @param binaryData array of byte to encode - * @return return encoded string - */ - static public String encode(byte[] binaryData) { - if (binaryData == null) { + public static String encode(byte[] bytes, boolean upperCase) { + if (bytes == null) { return null; } - - int lengthData = binaryData.length; - int lengthEncode = lengthData * 2; - char[] encodedData = new char[lengthEncode]; - int temp; - for (int i = 0; i < lengthData; i++) { - temp = binaryData[i]; - if (temp < 0) { - temp += 256; - } - encodedData[i*2] = lookUpHexAlphabet[temp >> 4]; - encodedData[i*2+1] = lookUpHexAlphabet[temp & 0xf]; + + final char[] chars = upperCase ? upperChars : lowerChars; + + char[] hex = new char[bytes.length * 2]; + for (int i = 0; i < bytes.length; i++) { + int b = bytes[i] & 0xFF; + hex[i * 2] = chars[b >> 4]; + hex[i * 2 + 1] = chars[b & 0xf]; } - return new String(encodedData); + return new String(hex); } /** * Decode hex string to a byte array - * + * * @param encoded encoded string * @return return array of byte to encode */ @@ -91,7 +84,7 @@ public final class HexBin { if (encoded == null) { return null; } - + int lengthData = encoded.length(); if (lengthData % 2 != 0) { return null; @@ -102,18 +95,18 @@ public final class HexBin { byte[] decodedData = new byte[lengthDecode]; byte temp1, temp2; char tempChar; - for( int i = 0; i