gian/phpmyadmin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #11354 from udan11/fix_lint_utf8

Browse Source 
Fix UTF strings in linter.
This commit is contained in:
Marc Delisle 2015-08-05 13:38:32 -04:00
commit 736ac22c5d
7 changed files with 166 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
libraries/Linter.class.php
View File

@ -26,20 +26,32 @@ class PMA_Linter
*/
public static function getLines($str)
{
$lines = array(0);
if ((!($str instanceof UtfString))
&& (defined('USE_UTF_STRINGS')) && (USE_UTF_STRINGS)
) {
// If the lexer uses UtfString for processing then the position will
// represent the position of the character and not the position of
// the byte.
$str = new SqlParser\UtfString($str);
}
// The reason for using the '8bit' parameter is that the length
// required is the length in bytes, not characters.
//
// Given the following string: `????+`, where `?` represents a
// multi-byte character (lets assume that every `?` is a 2-byte
// character) and `+` is a newline, the first value of `$i` is `0` and
// the last one is `4` (because there are 5 characters). Bytes `$str[0]`
// and `$str[1]` are the first character, `$str[2]` and `$str[3]` are
// the second one and `$str[4]` is going to be the first byte of the
// third character. The fourth and the last one (which is actually a new
// line) aren't going to be processed at all.
for ($i = 0, $len = /*overload*/mb_strlen($str, '8bit'); $i < $len; ++$i) {
// character) and `+` is a newline, the first value of `$i` is `0`
// and the last one is `4` (because there are 5 characters). Bytes
// `$str[0]` and `$str[1]` are the first character, `$str[2]` and
// `$str[3]` are the second one and `$str[4]` is going to be the
// first byte of the third character. The fourth and the last one
// (which is actually a new line) aren't going to be processed at
// all.
$len = ($str instanceof SqlParser\UtfString) ?
$str->length() : mb_strlen($len, '8bit');
$lines = array(0);
for ($i = 0; $i < $len; ++$i) {
if ($str[$i] === "\n") {
$lines[] = $i + 1;
}

4
libraries/sql-parser/src/Components/AlterOperation.php
View File

@ -179,7 +179,9 @@ class AlterOperation extends Component
} elseif ($token->value === ')') {
--$brackets;
} elseif ($token->value === ',') {
break;
if ($brackets === 0) {
break;
}
}
}
$ret->unknown[] = $token;

14
libraries/sql-parser/src/Context.php
View File

@ -297,13 +297,13 @@ abstract class Context
$len = strlen($str);
if ($str[0] === '#') {
return Token::FLAG_COMMENT_BASH;
} elseif (($len > 1) && ((($str[0] === '/') && ($str[1] === '*'))
|| (($str[0] === '*') && ($str[1] === '/')))
) {
} elseif (($len > 1) && ($str[0] === '/') && ($str[1] === '*')) {
return (($len > 2) && ($str[2] == '!')) ?
Token::FLAG_COMMENT_MYSQL_CMD : Token::FLAG_COMMENT_C;
} elseif (($len > 1) && ($str[0] === '*') && ($str[1] === '/')) {
return Token::FLAG_COMMENT_C;
} elseif (($len > 2) && ($str[0] === '-')
&& ($str[1] === '-') && ($str[2] !== "\n")
&& (static::isWhitespace($str[2]))
&& ($str[1] === '-') && (static::isWhitespace($str[2]))
) {
return Token::FLAG_COMMENT_SQL;
}
@ -416,10 +416,6 @@ abstract class Context
*/
public static function load($context = '')
{
/**
* @var Context $context
*/
if (empty($context)) {
$context = self::$defaultContext;
}

121
libraries/sql-parser/src/Lexer.php
View File

@ -63,37 +63,38 @@ namespace SqlParser {
{
/**
* A list of methods that are used in lexing the SQL query.
*
* @var array
*/
* A list of methods that are used in lexing the SQL query.
*
* @var array
*/
public static $PARSER_METHODS = array(
// It is best to put the parsers in order of their complexity
// (ascending) and their occurrence rate (descending).
//
// Conflicts:
//
// 1. `parseDelimiter` and `parseUnknown`, `parseKeyword`, `parseNumber`
// They fight over delimiter. The delimiter may be a keyword, a number
// or almost any character which makes the delimiter one of the first
// tokens that must be parsed.
//
// 1. `parseNumber` and `parseOperator`
// They fight over `+` and `-`.
//
// 2. `parseComment` and `parseOperator`
// They fight over `/` (as in ```/*comment*/``` or ```a / b```)
//
// 3. `parseBool` and `parseKeyword`
// They fight over `TRUE` and `FALSE`.
//
// 4. `parseKeyword` and `parseUnknown`
// They fight over words. `parseUnknown` does not know about keywords.
// It is best to put the parsers in order of their complexity
// (ascending) and their occurrence rate (descending).
//
// Conflicts:
//
// 1. `parseDelimiter`, `parseUnknown`, `parseKeyword`, `parseNumber`
// They fight over delimiter. The delimiter may be a keyword, a
// number or almost any character which makes the delimiter one of
// the first tokens that must be parsed.
//
// 1. `parseNumber` and `parseOperator`
// They fight over `+` and `-`.
//
// 2. `parseComment` and `parseOperator`
// They fight over `/` (as in ```/*comment*/``` or ```a / b```)
//
// 3. `parseBool` and `parseKeyword`
// They fight over `TRUE` and `FALSE`.
//
// 4. `parseKeyword` and `parseUnknown`
// They fight over words. `parseUnknown` does not know about
// keywords.
'parseDelimiter', 'parseWhitespace', 'parseNumber', 'parseComment',
'parseOperator', 'parseBool', 'parseString', 'parseSymbol',
'parseKeyword', 'parseUnknown'
'parseDelimiter', 'parseWhitespace', 'parseNumber',
'parseComment', 'parseOperator', 'parseBool', 'parseString',
'parseSymbol', 'parseKeyword', 'parseUnknown'
);
/**
@ -149,7 +150,7 @@ namespace SqlParser {
*
* @var string
*/
public $delimiter = ';';
public $delimiter;
/**
* The length of the delimiter.
@ -160,7 +161,7 @@ namespace SqlParser {
*
* @var int
*/
public $delimiterLen = 1;
public $delimiterLen;
/**
* List of errors that occurred during lexing.
@ -178,10 +179,12 @@ namespace SqlParser {
/**
* Constructor.
*
* @param string|UtfString $str The query to be lexed.
* @param bool $strict Whether strict mode should be enabled or not.
* @param string|UtfString $str The query to be lexed.
* @param bool $strict Whether strict mode should be
* enabled or not.
* @param string $delimiter The delimiter to be used.
*/
public function __construct($str, $strict = false)
public function __construct($str, $strict = false, $delimiter = null)
{
// `strlen` is used instead of `mb_strlen` because the lexer needs to
// parse each byte of the input.
@ -201,11 +204,24 @@ namespace SqlParser {
$this->strict = $strict;
// Setting the delimiter.
$this->delimiter = static::$DEFAULT_DELIMITER;
$this->setDelimiter(
!empty($delimiter) ? $delimiter : static::$DEFAULT_DELIMITER
);
$this->lex();
}
/**
* Sets the delimiter.
*
* @param string $delimiter The new delimiter.
*/
public function setDelimiter($delimiter)
{
$this->delimiter = $delimiter;
$this->delimiterLen = strlen($delimiter);
}
/**
* Parses the string and extracts lexemes.
*
@ -490,7 +506,7 @@ namespace SqlParser {
while ((++$this->last < $this->len) && ($this->str[$this->last] !== "\n")) {
$token .= $this->str[$this->last];
}
$token .= $this->str[$this->last];
$token .= "\n"; // Adding the line ending.
return new Token($token, Token::TYPE_COMMENT, Token::FLAG_COMMENT_BASH);
}
@ -503,11 +519,14 @@ namespace SqlParser {
// It is a MySQL-specific command.
$flags |= Token::FLAG_COMMENT_MYSQL_CMD;
}
while ((++$this->last < $this->len) &&
(($this->str[$this->last - 1] !== '*') || ($this->str[$this->last] !== '/'))) {
while ((++$this->last < $this->len)
&& (($this->str[$this->last - 1] !== '*') || ($this->str[$this->last] !== '/'))
) {
$token .= $this->str[$this->last];
}
if ($this->last < $this->len) {
$token .= $this->str[$this->last];
}
$token .= $this->str[$this->last];
return new Token($token, Token::TYPE_COMMENT, $flags);
}
}
@ -516,14 +535,12 @@ namespace SqlParser {
if (++$this->last < $this->len) {
$token .= $this->str[$this->last];
if (Context::isComment($token)) {
// Checking if this comment did not end already (```--\n```).
if ($this->str[$this->last] !== "\n") {
// Checking if this comment did not end already (```--\n```).
while ((++$this->last < $this->len) && ($this->str[$this->last] !== "\n")) {
$token .= $this->str[$this->last];
}
if ($this->last < $this->len) {
$token .= $this->str[$this->last];
}
$token .= "\n"; // Adding the line ending.
}
return new Token($token, Token::TYPE_COMMENT, Token::FLAG_COMMENT_SQL);
}
@ -763,13 +780,17 @@ namespace SqlParser {
$token = '';
}
if (($str = $this->parseString('`')) === null) {
if (($str = static::parseUnknown()) === null) {
$this->error(
__('Variable name was expected.'),
$this->str[$this->last],
$this->last
);
$str = null;
if ($this->last < $this->len) {
if (($str = $this->parseString('`')) === null) {
if (($str = static::parseUnknown()) === null) {
$this->error(
__('Variable name was expected.'),
$this->str[$this->last],
$this->last
);
}
}
}
@ -807,7 +828,7 @@ namespace SqlParser {
{
$idx = 0;
while ($idx < $this->delimiterLen) {
while (($idx < $this->delimiterLen) && ($this->last + $idx < $this->len)) {
if ($this->delimiter[$idx] !== $this->str[$this->last + $idx]) {
return null;
}

2
libraries/sql-parser/src/Statement.php
View File

@ -262,7 +262,7 @@ abstract class Statement
// this statement it means it is a new statement, but no
// delimiter was found between them.
$parser->error(
__('A new statement was found, but no delimiter between them.'),
__('A new statement was found, but no delimiter between it and the previous one.'),
$token
);
break;

2
libraries/sql-parser/src/Utils/Formatter.php
View File

@ -277,7 +277,7 @@ class Formatter
if ($prev !== null) {
// Checking if a new clause started.
if (static::isClause($prev)) {
if (static::isClause($prev) !== false) {
$lastClause = $prev->value;
$formattedOptions = false;
}

65
libraries/sql-parser/src/Utils/Query.php
View File

@ -587,6 +587,10 @@ class Query
for ($i = $statement->first; $i <= $statement->last; ++$i) {
$token = $list->tokens[$i];
if ($token->type === Token::TYPE_COMMENT) {
continue;
}
if ($token->type === Token::TYPE_OPERATOR) {
if ($token->value === '(') {
++$brackets;
@ -708,4 +712,65 @@ class Query
return $ret;
}
/**
* Gets the first full statement in the query.
*
* @param string $query The query to be analyzed.
* @param string $delimiter The delimiter to be used.
*
* @return array Array containing the first full query, the
* remaining part of the query and the last
* delimiter.
*/
public static function getFirstStatement($query, $delimiter = null)
{
$lexer = new Lexer($query, false, $delimiter);
$list = $lexer->list;
/**
* Whether a full statement was found.
* @var bool
*/
$fullStatement = false;
/**
* The first full statement.
* @var string
*/
$statement = '';
for ($list->idx = 0; $list->idx < $list->count; ++$list->idx) {
$token = $list->tokens[$list->idx];
if (($token->type === Token::TYPE_COMMENT)
&& (!($token->flags & Token::FLAG_COMMENT_MYSQL_CMD))
) {
continue;
}
$statement .= $token->token;
if (($token->type === Token::TYPE_DELIMITER) && (!empty($token->token))) {
$delimiter = $token->token;
$fullStatement = true;
break;
}
}
// No statement was found so we return the entire query as being the
// remaining part.
if (!$fullStatement) {
return array(null, $query, $delimiter);
}
// At least one query was found so we have to build the rest of the
// remaining query.
$query = '';
for (++$list->idx; $list->idx < $list->count; ++$list->idx) {
$query .= $list->tokens[$list->idx]->token;
}
return array(trim($statement), $query, $delimiter);
}
}