From 13ff72c4d7dfd8c1ed6063d5647e5dfdfc8f3e9f Mon Sep 17 00:00:00 2001 From: Dan Ungureanu Date: Wed, 5 Aug 2015 19:01:51 +0300 Subject: [PATCH] Fix UTF strings in linter. Updated sql-parser to udan11/sql-parser@8a1011e97497cbecbfb7cf9ef36146aaf81a1486. Signed-off-by: Dan Ungureanu --- libraries/Linter.class.php | 28 ++-- .../src/Components/AlterOperation.php | 4 +- libraries/sql-parser/src/Context.php | 14 +- libraries/sql-parser/src/Lexer.php | 121 ++++++++++-------- libraries/sql-parser/src/Statement.php | 2 +- libraries/sql-parser/src/Utils/Formatter.php | 2 +- libraries/sql-parser/src/Utils/Query.php | 65 ++++++++++ 7 files changed, 166 insertions(+), 70 deletions(-) diff --git a/libraries/Linter.class.php b/libraries/Linter.class.php index a8ea1a2f5a..3b82767517 100644 --- a/libraries/Linter.class.php +++ b/libraries/Linter.class.php @@ -26,20 +26,32 @@ class PMA_Linter */ public static function getLines($str) { - $lines = array(0); + if ((!($str instanceof UtfString)) + && (defined('USE_UTF_STRINGS')) && (USE_UTF_STRINGS) + ) { + // If the lexer uses UtfString for processing then the position will + // represent the position of the character and not the position of + // the byte. + $str = new SqlParser\UtfString($str); + } // The reason for using the '8bit' parameter is that the length // required is the length in bytes, not characters. // // Given the following string: `????+`, where `?` represents a // multi-byte character (lets assume that every `?` is a 2-byte - // character) and `+` is a newline, the first value of `$i` is `0` and - // the last one is `4` (because there are 5 characters). Bytes `$str[0]` - // and `$str[1]` are the first character, `$str[2]` and `$str[3]` are - // the second one and `$str[4]` is going to be the first byte of the - // third character. The fourth and the last one (which is actually a new - // line) aren't going to be processed at all. - for ($i = 0, $len = /*overload*/mb_strlen($str, '8bit'); $i < $len; ++$i) { + // character) and `+` is a newline, the first value of `$i` is `0` + // and the last one is `4` (because there are 5 characters). Bytes + // `$str[0]` and `$str[1]` are the first character, `$str[2]` and + // `$str[3]` are the second one and `$str[4]` is going to be the + // first byte of the third character. The fourth and the last one + // (which is actually a new line) aren't going to be processed at + // all. + $len = ($str instanceof SqlParser\UtfString) ? + $str->length() : mb_strlen($len, '8bit'); + + $lines = array(0); + for ($i = 0; $i < $len; ++$i) { if ($str[$i] === "\n") { $lines[] = $i + 1; } diff --git a/libraries/sql-parser/src/Components/AlterOperation.php b/libraries/sql-parser/src/Components/AlterOperation.php index 9cae43c800..ae28d51587 100644 --- a/libraries/sql-parser/src/Components/AlterOperation.php +++ b/libraries/sql-parser/src/Components/AlterOperation.php @@ -179,7 +179,9 @@ class AlterOperation extends Component } elseif ($token->value === ')') { --$brackets; } elseif ($token->value === ',') { - break; + if ($brackets === 0) { + break; + } } } $ret->unknown[] = $token; diff --git a/libraries/sql-parser/src/Context.php b/libraries/sql-parser/src/Context.php index eb7f29976b..c52cbcf3fa 100644 --- a/libraries/sql-parser/src/Context.php +++ b/libraries/sql-parser/src/Context.php @@ -297,13 +297,13 @@ abstract class Context $len = strlen($str); if ($str[0] === '#') { return Token::FLAG_COMMENT_BASH; - } elseif (($len > 1) && ((($str[0] === '/') && ($str[1] === '*')) - || (($str[0] === '*') && ($str[1] === '/'))) - ) { + } elseif (($len > 1) && ($str[0] === '/') && ($str[1] === '*')) { + return (($len > 2) && ($str[2] == '!')) ? + Token::FLAG_COMMENT_MYSQL_CMD : Token::FLAG_COMMENT_C; + } elseif (($len > 1) && ($str[0] === '*') && ($str[1] === '/')) { return Token::FLAG_COMMENT_C; } elseif (($len > 2) && ($str[0] === '-') - && ($str[1] === '-') && ($str[2] !== "\n") - && (static::isWhitespace($str[2])) + && ($str[1] === '-') && (static::isWhitespace($str[2])) ) { return Token::FLAG_COMMENT_SQL; } @@ -416,10 +416,6 @@ abstract class Context */ public static function load($context = '') { - /** - * @var Context $context - */ - if (empty($context)) { $context = self::$defaultContext; } diff --git a/libraries/sql-parser/src/Lexer.php b/libraries/sql-parser/src/Lexer.php index 697218c969..69c1ff2091 100644 --- a/libraries/sql-parser/src/Lexer.php +++ b/libraries/sql-parser/src/Lexer.php @@ -63,37 +63,38 @@ namespace SqlParser { { /** - * A list of methods that are used in lexing the SQL query. - * - * @var array - */ + * A list of methods that are used in lexing the SQL query. + * + * @var array + */ public static $PARSER_METHODS = array( - // It is best to put the parsers in order of their complexity - // (ascending) and their occurrence rate (descending). - // - // Conflicts: - // - // 1. `parseDelimiter` and `parseUnknown`, `parseKeyword`, `parseNumber` - // They fight over delimiter. The delimiter may be a keyword, a number - // or almost any character which makes the delimiter one of the first - // tokens that must be parsed. - // - // 1. `parseNumber` and `parseOperator` - // They fight over `+` and `-`. - // - // 2. `parseComment` and `parseOperator` - // They fight over `/` (as in ```/*comment*/``` or ```a / b```) - // - // 3. `parseBool` and `parseKeyword` - // They fight over `TRUE` and `FALSE`. - // - // 4. `parseKeyword` and `parseUnknown` - // They fight over words. `parseUnknown` does not know about keywords. + // It is best to put the parsers in order of their complexity + // (ascending) and their occurrence rate (descending). + // + // Conflicts: + // + // 1. `parseDelimiter`, `parseUnknown`, `parseKeyword`, `parseNumber` + // They fight over delimiter. The delimiter may be a keyword, a + // number or almost any character which makes the delimiter one of + // the first tokens that must be parsed. + // + // 1. `parseNumber` and `parseOperator` + // They fight over `+` and `-`. + // + // 2. `parseComment` and `parseOperator` + // They fight over `/` (as in ```/*comment*/``` or ```a / b```) + // + // 3. `parseBool` and `parseKeyword` + // They fight over `TRUE` and `FALSE`. + // + // 4. `parseKeyword` and `parseUnknown` + // They fight over words. `parseUnknown` does not know about + // keywords. - 'parseDelimiter', 'parseWhitespace', 'parseNumber', 'parseComment', - 'parseOperator', 'parseBool', 'parseString', 'parseSymbol', - 'parseKeyword', 'parseUnknown' + 'parseDelimiter', 'parseWhitespace', 'parseNumber', + 'parseComment', 'parseOperator', 'parseBool', 'parseString', + 'parseSymbol', 'parseKeyword', 'parseUnknown' ); /** @@ -149,7 +150,7 @@ namespace SqlParser { * * @var string */ - public $delimiter = ';'; + public $delimiter; /** * The length of the delimiter. @@ -160,7 +161,7 @@ namespace SqlParser { * * @var int */ - public $delimiterLen = 1; + public $delimiterLen; /** * List of errors that occurred during lexing. @@ -178,10 +179,12 @@ namespace SqlParser { /** * Constructor. * - * @param string|UtfString $str The query to be lexed. - * @param bool $strict Whether strict mode should be enabled or not. + * @param string|UtfString $str The query to be lexed. + * @param bool $strict Whether strict mode should be + * enabled or not. + * @param string $delimiter The delimiter to be used. */ - public function __construct($str, $strict = false) + public function __construct($str, $strict = false, $delimiter = null) { // `strlen` is used instead of `mb_strlen` because the lexer needs to // parse each byte of the input. @@ -201,11 +204,24 @@ namespace SqlParser { $this->strict = $strict; // Setting the delimiter. - $this->delimiter = static::$DEFAULT_DELIMITER; + $this->setDelimiter( + !empty($delimiter) ? $delimiter : static::$DEFAULT_DELIMITER + ); $this->lex(); } + /** + * Sets the delimiter. + * + * @param string $delimiter The new delimiter. + */ + public function setDelimiter($delimiter) + { + $this->delimiter = $delimiter; + $this->delimiterLen = strlen($delimiter); + } + /** * Parses the string and extracts lexemes. * @@ -490,7 +506,7 @@ namespace SqlParser { while ((++$this->last < $this->len) && ($this->str[$this->last] !== "\n")) { $token .= $this->str[$this->last]; } - $token .= $this->str[$this->last]; + $token .= "\n"; // Adding the line ending. return new Token($token, Token::TYPE_COMMENT, Token::FLAG_COMMENT_BASH); } @@ -503,11 +519,14 @@ namespace SqlParser { // It is a MySQL-specific command. $flags |= Token::FLAG_COMMENT_MYSQL_CMD; } - while ((++$this->last < $this->len) && - (($this->str[$this->last - 1] !== '*') || ($this->str[$this->last] !== '/'))) { + while ((++$this->last < $this->len) + && (($this->str[$this->last - 1] !== '*') || ($this->str[$this->last] !== '/')) + ) { + $token .= $this->str[$this->last]; + } + if ($this->last < $this->len) { $token .= $this->str[$this->last]; } - $token .= $this->str[$this->last]; return new Token($token, Token::TYPE_COMMENT, $flags); } } @@ -516,14 +535,12 @@ namespace SqlParser { if (++$this->last < $this->len) { $token .= $this->str[$this->last]; if (Context::isComment($token)) { + // Checking if this comment did not end already (```--\n```). if ($this->str[$this->last] !== "\n") { - // Checking if this comment did not end already (```--\n```). while ((++$this->last < $this->len) && ($this->str[$this->last] !== "\n")) { $token .= $this->str[$this->last]; } - if ($this->last < $this->len) { - $token .= $this->str[$this->last]; - } + $token .= "\n"; // Adding the line ending. } return new Token($token, Token::TYPE_COMMENT, Token::FLAG_COMMENT_SQL); } @@ -763,13 +780,17 @@ namespace SqlParser { $token = ''; } - if (($str = $this->parseString('`')) === null) { - if (($str = static::parseUnknown()) === null) { - $this->error( - __('Variable name was expected.'), - $this->str[$this->last], - $this->last - ); + $str = null; + + if ($this->last < $this->len) { + if (($str = $this->parseString('`')) === null) { + if (($str = static::parseUnknown()) === null) { + $this->error( + __('Variable name was expected.'), + $this->str[$this->last], + $this->last + ); + } } } @@ -807,7 +828,7 @@ namespace SqlParser { { $idx = 0; - while ($idx < $this->delimiterLen) { + while (($idx < $this->delimiterLen) && ($this->last + $idx < $this->len)) { if ($this->delimiter[$idx] !== $this->str[$this->last + $idx]) { return null; } diff --git a/libraries/sql-parser/src/Statement.php b/libraries/sql-parser/src/Statement.php index fabb82d7f1..8bf3d24ef2 100644 --- a/libraries/sql-parser/src/Statement.php +++ b/libraries/sql-parser/src/Statement.php @@ -262,7 +262,7 @@ abstract class Statement // this statement it means it is a new statement, but no // delimiter was found between them. $parser->error( - __('A new statement was found, but no delimiter between them.'), + __('A new statement was found, but no delimiter between it and the previous one.'), $token ); break; diff --git a/libraries/sql-parser/src/Utils/Formatter.php b/libraries/sql-parser/src/Utils/Formatter.php index 5d590c3b65..dc8c3af31f 100644 --- a/libraries/sql-parser/src/Utils/Formatter.php +++ b/libraries/sql-parser/src/Utils/Formatter.php @@ -277,7 +277,7 @@ class Formatter if ($prev !== null) { // Checking if a new clause started. - if (static::isClause($prev)) { + if (static::isClause($prev) !== false) { $lastClause = $prev->value; $formattedOptions = false; } diff --git a/libraries/sql-parser/src/Utils/Query.php b/libraries/sql-parser/src/Utils/Query.php index c11e858bb7..604a902418 100644 --- a/libraries/sql-parser/src/Utils/Query.php +++ b/libraries/sql-parser/src/Utils/Query.php @@ -587,6 +587,10 @@ class Query for ($i = $statement->first; $i <= $statement->last; ++$i) { $token = $list->tokens[$i]; + if ($token->type === Token::TYPE_COMMENT) { + continue; + } + if ($token->type === Token::TYPE_OPERATOR) { if ($token->value === '(') { ++$brackets; @@ -708,4 +712,65 @@ class Query return $ret; } + + /** + * Gets the first full statement in the query. + * + * @param string $query The query to be analyzed. + * @param string $delimiter The delimiter to be used. + * + * @return array Array containing the first full query, the + * remaining part of the query and the last + * delimiter. + */ + public static function getFirstStatement($query, $delimiter = null) + { + $lexer = new Lexer($query, false, $delimiter); + $list = $lexer->list; + + /** + * Whether a full statement was found. + * @var bool + */ + $fullStatement = false; + + /** + * The first full statement. + * @var string + */ + $statement = ''; + + for ($list->idx = 0; $list->idx < $list->count; ++$list->idx) { + $token = $list->tokens[$list->idx]; + + if (($token->type === Token::TYPE_COMMENT) + && (!($token->flags & Token::FLAG_COMMENT_MYSQL_CMD)) + ) { + continue; + } + + $statement .= $token->token; + + if (($token->type === Token::TYPE_DELIMITER) && (!empty($token->token))) { + $delimiter = $token->token; + $fullStatement = true; + break; + } + } + + // No statement was found so we return the entire query as being the + // remaining part. + if (!$fullStatement) { + return array(null, $query, $delimiter); + } + + // At least one query was found so we have to build the rest of the + // remaining query. + $query = ''; + for (++$list->idx; $list->idx < $list->count; ++$list->idx) { + $query .= $list->tokens[$list->idx]->token; + } + + return array(trim($statement), $query, $delimiter); + } }