From 53cff80649296475ab526f7cf7eae22ba31178d4 Mon Sep 17 00:00:00 2001
From: Louis Xu <rdxiaolei@gmail.com>
Date: Mon, 28 Mar 2016 19:25:27 +0800
Subject: [PATCH] Add Lexer and Tokens for Teradata

---
 .../teradata/parser/TeradataLexer.java        | 120 +++++++++++++++++-
 .../com/alibaba/druid/sql/parser/Token.java   |  10 ++
 2 files changed, 127 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/alibaba/druid/sql/dialect/teradata/parser/TeradataLexer.java b/src/main/java/com/alibaba/druid/sql/dialect/teradata/parser/TeradataLexer.java
index ebd7d2e530..42bbd60dc1 100644
--- a/src/main/java/com/alibaba/druid/sql/dialect/teradata/parser/TeradataLexer.java
+++ b/src/main/java/com/alibaba/druid/sql/dialect/teradata/parser/TeradataLexer.java
@@ -15,26 +15,140 @@
  */
 package com.alibaba.druid.sql.dialect.teradata.parser;
 
+import static com.alibaba.druid.sql.parser.LayoutCharacters.EOI;
+
 import java.util.HashMap;
 import java.util.Map;
 
 import com.alibaba.druid.sql.parser.Keywords;
 import com.alibaba.druid.sql.parser.Lexer;
+import com.alibaba.druid.sql.parser.NotAllowCommentException;
 import com.alibaba.druid.sql.parser.Token;
 
 public class TeradataLexer extends Lexer {
-    public final static Keywords DEFAULT_PG_KEYWORDS;
+    public final static Keywords DEFAULT_TD_KEYWORDS;
 
     static {
         Map<String, Token> map = new HashMap<String, Token>();
         map.putAll(Keywords.DEFAULT_KEYWORDS.getKeywords());
         
-        DEFAULT_PG_KEYWORDS = new Keywords(map);
+        map.put("SEL", Token.SEL);
+        map.put("LOCKING", Token.LOCKING);
+        map.put("ACCESS", Token.ACCESS);
+        
+//        map.put("PARTITION", Token.PARTITION);
+//        map.put("PARTITIONED", Token.PARTITIONED);
+        map.put("RANGE_N", Token.RANGE_N);
+        
+        map.put("VOLATILE", Token.VOLATILE);
+        map.put("MULTISET", Token.MULTISET);
+        
+        map.put("FORMAT", Token.FORMAT);
+        
+        DEFAULT_TD_KEYWORDS = new Keywords(map);
     }
     
     public TeradataLexer(String input){
         super(input);
-        super.keywods = DEFAULT_PG_KEYWORDS;
+        super.keywods = DEFAULT_TD_KEYWORDS;
+    }
+    
+    public void scanComment() {
+        if (ch != '/' && ch != '-') {
+            throw new IllegalStateException();
+        }
+
+        mark = pos;
+        bufPos = 0;
+        scanChar();
+
+        // /*+ */
+        if (ch == '*') {
+            scanChar();
+            bufPos++;
+
+            while (ch == ' ') {
+                scanChar();
+                bufPos++;
+            }
+
+            boolean isHint = false;
+            int startHintSp = bufPos + 1;
+            if (ch == '+') {
+                isHint = true;
+                scanChar();
+                bufPos++;
+            }
+
+            for (;;) {
+                if (ch == '*' && charAt(pos + 1) == '/') {
+                    bufPos += 2;
+                    scanChar();
+                    scanChar();
+                    break;
+                }
+
+                scanChar();
+                bufPos++;
+            }
+
+            if (isHint) {
+                stringVal = subString(mark + startHintSp, (bufPos - startHintSp) - 1);
+                token = Token.HINT;
+            } else {
+                stringVal = subString(mark, bufPos);
+                token = Token.MULTI_LINE_COMMENT;
+                if (keepComments) {
+                    addComment(stringVal);
+                }
+            }
+
+            if (token != Token.HINT && !isAllowComment()) {
+                throw new NotAllowCommentException();
+            }
+
+            return;
+        }
+
+        if (!isAllowComment()) {
+            throw new NotAllowCommentException();
+        }
+
+        if (ch == '/' || ch == '-') {
+            scanChar();
+            bufPos++;
+
+            for (;;) {
+                if (ch == '\r') {
+                    if (charAt(pos + 1) == '\n') {
+                        bufPos += 2;
+                        scanChar();
+                        break;
+                    }
+                    bufPos++;
+                    break;
+                } else if (ch == EOI) {
+                    break;
+                }
+
+                if (ch == '\n') {
+                    scanChar();
+                    bufPos++;
+                    break;
+                }
+
+                scanChar();
+                bufPos++;
+            }
+
+            stringVal = subString(mark + 1, bufPos);
+            token = Token.LINE_COMMENT;
+            if (keepComments) {
+                addComment(stringVal);
+            }
+            endOfComment = isEOF();
+            return;
+        }
     }
 
 }
diff --git a/src/main/java/com/alibaba/druid/sql/parser/Token.java b/src/main/java/com/alibaba/druid/sql/parser/Token.java
index a623d8a7ca..7f5dd6f188 100644
--- a/src/main/java/com/alibaba/druid/sql/parser/Token.java
+++ b/src/main/java/com/alibaba/druid/sql/parser/Token.java
@@ -261,6 +261,16 @@ public enum Token {
     PARTITION,
     PARTITIONED,
     OVERWRITE,
+    
+    // Teradata
+    SEL("SEL"),
+    LOCKING("LOCKING"),
+    ACCESS("ACCESS"),
+    VOLATILE("VOLATILE"),
+    MULTISET("MULTISET"),
+    POSITION("POSITION"),
+    RANGE_N("RANGE_N"),
+    FORMAT("FORMAT"),
 
     LPAREN("("), 
     RPAREN(")"),