From 762d67dc363037be2c5eb5f4925a9968dc4dec18 Mon Sep 17 00:00:00 2001 From: Dan Ungureanu Date: Sun, 9 Aug 2015 22:16:51 +0300 Subject: [PATCH] Rewrote a part of the import mechanism. Updated sql-parser library to udan11/sql-parser@e713c9fa1d79fd7bc90c63016f9f3bff8f983794. Signed-off-by: Dan Ungureanu --- libraries/plugins/import/ImportSql.class.php | 490 ++---------------- .../sql-parser/src/Components/JoinKeyword.php | 2 +- .../src/Components/OptionsArray.php | 8 +- libraries/sql-parser/src/Lexer.php | 2 +- libraries/sql-parser/src/Parser.php | 6 +- .../src/Statements/AlterStatement.php | 2 +- libraries/sql-parser/src/Token.php | 2 +- .../sql-parser/src/Utils/BufferedQuery.php | 34 +- libraries/sql-parser/src/Utils/Query.php | 6 +- test/classes/plugin/import/ImportSql_test.php | 2 +- 10 files changed, 69 insertions(+), 485 deletions(-) diff --git a/libraries/plugins/import/ImportSql.class.php b/libraries/plugins/import/ImportSql.class.php index 8226b57fcc..d09ae01bb0 100644 --- a/libraries/plugins/import/ImportSql.class.php +++ b/libraries/plugins/import/ImportSql.class.php @@ -21,103 +21,6 @@ require_once 'libraries/plugins/ImportPlugin.class.php'; */ class ImportSql extends ImportPlugin { - const BIG_VALUE = 2147483647; - const READ_MB_FALSE = 0; - const READ_MB_TRUE = 1; - - /** - * @var string SQL delimiter - */ - private $_delimiter; - - /** - * @var int SQL delimiter length - */ - private $_delimiterLength; - - /** - * @var bool|int SQL delimiter position or false if not found - */ - private $_delimiterPosition = false; - - /** - * @var int Query start position - */ - private $_queryBeginPosition = 0; - - /** - * @var int|false First special chars position or false if not found - */ - private $_firstSearchChar = null; - - /** - * @var bool Current position is in string - */ - private $_isInString = false; - - /** - * @var string Quote of current string or null if out of string - */ - private $_quote = null; - - /** - * @var bool Current position is in comment - */ - private $_isInComment = false; - - /** - * @var string Current comment opener - */ - private $_openingComment = null; - - /** - * @var bool Current position is in delimiter definition - */ - private $_isInDelimiter = false; - - /** - * @var string Delimiter keyword - */ - private $_delimiterKeyword = 'DELIMITER '; - - /** - * @var int Import should be done using multibytes - */ - private $_readMb = self::READ_MB_FALSE; - - /** - * @var string Data to parse - */ - private $_data = null; - - /** - * @var int Length of data to parse - */ - private $_dataLength = 0; - - /** - * @var array List of string functions - * @todo Move this part in string functions definition file. - */ - private $_stringFunctions = array( - self::READ_MB_FALSE => array( - 'substr' => 'substr', - 'strlen' => 'strlen', - 'strpos' => 'strpos', - 'strtoupper' => 'strtoupper', - ), - self::READ_MB_TRUE => array( - 'substr' => 'mb_substr', - 'strlen' => 'mb_strlen', - 'strpos' => 'mb_strpos', - 'strtoupper' => 'mb_strtoupper', - ), - ); - - /** - * @var bool|int List of string functions to use - */ - private $_stringFctToUse = false; /** * Constructor @@ -189,13 +92,6 @@ class ImportSql extends ImportPlugin ); $generalOptions->addProperty($leaf); - $leaf = new BoolPropertyItem(); - $leaf->setName("read_as_multibytes"); - $leaf->setText( - __('Read as multibytes') - ); - $generalOptions->addProperty($leaf); - // add the main group to the root group $importSpecificOptions->addProperty($generalOptions); // set the options for the import plugin property item @@ -205,182 +101,7 @@ class ImportSql extends ImportPlugin $this->properties = $importPluginProperties; } - /** - * Look for end of string - * - * @return bool End of string found - */ - private function _searchStringEnd() - { - //Search for closing quote - $posClosingString = $this->_stringFctToUse['strpos']( - $this->_data, $this->_quote, $this->_delimiterPosition - ); - - if (false === $posClosingString) { - return false; - } - - //Quotes escaped by quote will be considered as 2 consecutive strings - //and won't pass in this loop. - $posEscape = $posClosingString-1; - while ($this->_stringFctToUse['substr']($this->_data, $posEscape, 1) == '\\' - ) { - $posEscape--; - } - - // Odd count means it was escaped - $quoteEscaped = (((($posClosingString - 1) - $posEscape) % 2) === 1); - - //Move after the escaped quote. - $this->_delimiterPosition = $posClosingString + 1; - - if ($quoteEscaped) { - return true; - } - - $this->_isInString = false; - $this->_quote = null; - return true; - } - - /** - * Return the position of first SQL delimiter or false if no SQL delimiter found. - * - * @return int|bool Delimiter position or false if no delimiter found - */ - private function _findDelimiterPosition() - { - $this->_firstSearchChar = null; - $firstSqlDelimiter = null; - $matches = null; - - /* while not at end of line */ - while ($this->_delimiterPosition < $this->_dataLength) { - if ($this->_isInString) { - if (false === $this->_searchStringEnd()) { - return false; - } - - continue; - } - - if ($this->_isInComment) { - if (in_array($this->_openingComment, array('#', '-- '))) { - $posClosingComment = $this->_stringFctToUse['strpos']( - $this->_data, - "\n", - $this->_delimiterPosition - ); - if (false === $posClosingComment) { - return false; - } - //Move after the end of the line. - $this->_delimiterPosition = $posClosingComment + 1; - $this->_isInComment = false; - $this->_openingComment = null; - } elseif ('/*' === $this->_openingComment) { - //Search for closing comment - $posClosingComment = $this->_stringFctToUse['strpos']( - $this->_data, - '*/', - $this->_delimiterPosition - ); - if (false === $posClosingComment) { - return false; - } - //Move after closing comment. - $this->_delimiterPosition = $posClosingComment + 2; - $this->_isInComment = false; - $this->_openingComment = null; - } else { - //We shouldn't be able to come here. - //throw new Exception('Unknown case.'); - break; - } - - continue; - } - - if ($this->_isInDelimiter) { - //Search for new line. - if (!preg_match( - "/^(.*)\n/", - $this->_stringFctToUse['substr']( - $this->_data, - $this->_delimiterPosition - ), - $matches, - PREG_OFFSET_CAPTURE - )) { - return false; - } - - $this->_setDelimiter($matches[1][0]); - //Start after delimiter and new line. - $this->_queryBeginPosition = $this->_delimiterPosition - + $matches[1][1] + $this->_delimiterLength + 1; - $this->_delimiterPosition = $this->_queryBeginPosition; - $this->_isInDelimiter = false; - $firstSqlDelimiter = null; - $this->_firstSearchChar = null; - continue; - } - - $matches = $this->_searchSpecialChars($matches); - - $firstSqlDelimiter = $this->_searchSqlDelimiter($firstSqlDelimiter); - - if (false === $firstSqlDelimiter && false === $this->_firstSearchChar) { - return false; - } - - //If first char is delimiter. - if (false === $this->_firstSearchChar - || (false !== $firstSqlDelimiter - && $firstSqlDelimiter < $this->_firstSearchChar) - ) { - $this->_delimiterPosition = $firstSqlDelimiter; - return true; - } - - //Else first char is result of preg_match. - - $specialChars = $matches[1][0]; - - //If string is opened. - if (in_array($specialChars, array('\'', '"', '`'))) { - $this->_isInString = true; - $this->_quote = $specialChars; - //Move before quote. - $this->_delimiterPosition = $this->_firstSearchChar + 1; - continue; - } - - //If comment is opened. - if (in_array($specialChars, array('#', '-- ', '/*'))) { - $this->_isInComment = true; - $this->_openingComment = $specialChars; - //Move before comment opening. - $this->_delimiterPosition = $this->_firstSearchChar - + $this->_stringFctToUse['strlen']($specialChars); - continue; - } - - //If DELIMITER is found. - $specialCharsUpper = $this->_stringFctToUse['strtoupper']($specialChars); - if ($specialCharsUpper === $this->_delimiterKeyword) { - $this->_isInDelimiter = true; - $this->_delimiterPosition = $this->_firstSearchChar - + $this->_stringFctToUse['strlen']($specialChars); - continue; - } - } - - return false; - } - - /** + /* * Handles the whole import logic * * @param array &$sql_data 2-element array with sql data @@ -391,95 +112,63 @@ class ImportSql extends ImportPlugin { global $error, $timeout_passed; - //Manage multibytes or not - if (isset($_REQUEST['sql_read_as_multibytes'])) { - $this->_readMb = self::READ_MB_TRUE; - } - $this->_stringFctToUse = $this->_stringFunctions[$this->_readMb]; - - if (isset($_POST['sql_delimiter'])) { - $this->_setDelimiter($_POST['sql_delimiter']); - } else { - $this->_setDelimiter(';'); - } - - // Handle compatibility options + // Handle compatibility options. $this->_setSQLMode($GLOBALS['dbi'], $_REQUEST); - //Initialise data. - $this->_setData(null); + $bq = new SqlParser\Utils\BufferedQuery(); + if (isset($_POST['sql_delimiter'])) { + $bq->setDelimiter($_POST['sql_delimiter']); + } /** - * will be set in PMA_importGetNextChunk() - * - * @global boolean $GLOBALS['finished'] + * Will be set in PMA_importGetNextChunk(). + * @global bool $GLOBALS['finished'] */ $GLOBALS['finished'] = false; - $delimiterFound = false; - while (!$error && !$timeout_passed) { - if (false === $delimiterFound) { + while ((!$error) && (!$timeout_passed)) { + + // Getting the first statement, the remaining data and the last + // delimiter. + $statement = $bq->extract(); + + // If there is no full statement, we are looking for more data. + if (empty($statement)) { + + // Importing new data. $newData = PMA_importGetNextChunk(200); + + // Subtract data we didn't handle yet and stop processing. if ($newData === false) { - // subtract data we didn't handle yet and stop processing - $GLOBALS['offset'] -= $this->_dataLength; + $GLOBALS['offset'] -= mb_strlen($bq->query); break; } + // Checking if the input buffer has finished. if ($newData === true) { $GLOBALS['finished'] = true; break; } - //Convert CR (but not CRLF) to LF otherwise all queries - //may not get executed on some platforms - $this->_addData(preg_replace("/\r($|[^\n])/", "\n$1", $newData)); - unset($newData); - } + // Convert CR (but not CRLF) to LF otherwise all queries may + // not get executed on some platforms. + $bq->query .= preg_replace("/\r($|[^\n])/", "\n$1", $newData); - //Find quotes, comments, delimiter definition or delimiter itself. - $delimiterFound = $this->_findDelimiterPosition(); - - //If no delimiter found, restart and get more data. - if (false === $delimiterFound) { continue; } - PMA_importRunQuery( - $this->_stringFctToUse['substr']( - $this->_data, - $this->_queryBeginPosition, - $this->_delimiterPosition - $this->_queryBeginPosition - ), //Query to execute - $this->_stringFctToUse['substr']( - $this->_data, - 0, - $this->_delimiterPosition + $this->_delimiterLength - ), //Query to display - false, - $sql_data - ); - - $this->_setData( - $this->_stringFctToUse['substr']( - $this->_data, - $this->_delimiterPosition + $this->_delimiterLength - ) - ); + // Executing the query. + PMA_importRunQuery($statement, $statement, false, $sql_data); } - if (! $timeout_passed) { - //Commit any possible data in buffers - PMA_importRunQuery( - $this->_stringFctToUse['substr']( - $this->_data, - $this->_queryBeginPosition - ), //Query to execute - $this->_data, - false, - $sql_data - ); + // Extracting remaining statements. + while ((!$error) && (!$timeout_passed) + && ($statement = $bq->extract(true)) + ) { + PMA_importRunQuery($statement, $statement, false, $sql_data); } + + // Finishing. PMA_importRunQuery('', '', false, $sql_data); } @@ -508,113 +197,4 @@ class ImportSql extends ImportPlugin ); } } - - /** - * Look for special chars: comment, string or DELIMITER - * - * @param array $matches Special chars found in data - * - * @return array matches - */ - private function _searchSpecialChars( - $matches - ) { - //Don't look for a string/comment/"DELIMITER" if not found previously - //or if it's still after current position. - if (null === $this->_firstSearchChar - || (false !== $this->_firstSearchChar - && $this->_firstSearchChar < $this->_delimiterPosition) - ) { - $bFind = preg_match( - '/(\'|"|#|-- |\/\*|`|(?i)(?_delimiterKeyword . ')/', - $this->_stringFctToUse['substr']( - $this->_data, - $this->_delimiterPosition - ), - $matches, - PREG_OFFSET_CAPTURE - ); - - if (1 === $bFind) { - $this->_firstSearchChar = $matches[1][1] + $this->_delimiterPosition; - } else { - $this->_firstSearchChar = false; - } - } - return $matches; - } - - /** - * Look for SQL delimiter - * - * @param int $firstSqlDelimiter First found char position - * - * @return int - */ - private function _searchSqlDelimiter($firstSqlDelimiter) - { - //Don't look for the SQL delimiter if not found previously - //or if it's still after current position. - if (null === $firstSqlDelimiter - || (false !== $firstSqlDelimiter - && $firstSqlDelimiter < $this->_delimiterPosition) - ) { - // the cost of doing this one with preg_match() would be too high - $firstSqlDelimiter = $this->_stringFctToUse['strpos']( - $this->_data, - $this->_delimiter, - $this->_delimiterPosition - ); - } - - return $firstSqlDelimiter; - } - - /** - * Set new delimiter - * - * @param string $delimiter New delimiter - * - * @return int delimiter length - */ - private function _setDelimiter($delimiter) - { - $this->_delimiter = $delimiter; - $this->_delimiterLength = $this->_stringFctToUse['strlen']($delimiter); - - return $this->_delimiterLength; - } - - /** - * Set data to parse - * - * @param string $data Data to parse - * - * @return int Data length - */ - private function _setData($data) - { - $this->_data = ltrim($data); - $this->_dataLength = $this->_stringFctToUse['strlen']($this->_data); - $this->_queryBeginPosition = 0; - $this->_delimiterPosition = 0; - - return $this->_dataLength; - } - - /** - * Add data to parse - * - * @param string $data Data to add to data to parse - * - * @return int Data length - */ - private function _addData($data) - { - $this->_data .= $data; - $this->_dataLength += $this->_stringFctToUse['strlen']($data); - - return $this->_dataLength; - } -} +} \ No newline at end of file diff --git a/libraries/sql-parser/src/Components/JoinKeyword.php b/libraries/sql-parser/src/Components/JoinKeyword.php index a647dcdf9f..171ad98db3 100644 --- a/libraries/sql-parser/src/Components/JoinKeyword.php +++ b/libraries/sql-parser/src/Components/JoinKeyword.php @@ -71,7 +71,7 @@ class JoinKeyword extends Component { $ret = array(); - $expr = new JoinKeyword();; + $expr = new JoinKeyword(); /** * The state of the parser. diff --git a/libraries/sql-parser/src/Components/OptionsArray.php b/libraries/sql-parser/src/Components/OptionsArray.php index 0e863266db..da2ec8dbee 100644 --- a/libraries/sql-parser/src/Components/OptionsArray.php +++ b/libraries/sql-parser/src/Components/OptionsArray.php @@ -55,10 +55,10 @@ class OptionsArray extends Component { $ret = new OptionsArray(); - /** - * The ID that will be assigned to duplicate options. - * @var int $lastAssignedId - */ + /** + * The ID that will be assigned to duplicate options. + * @var int $lastAssignedId + */ $lastAssignedId = count($options) + 1; /** diff --git a/libraries/sql-parser/src/Lexer.php b/libraries/sql-parser/src/Lexer.php index 82385e309c..b969a1959a 100644 --- a/libraries/sql-parser/src/Lexer.php +++ b/libraries/sql-parser/src/Lexer.php @@ -506,7 +506,7 @@ namespace SqlParser { while ((++$this->last < $this->len) && ($this->str[$this->last] !== "\n")) { $token .= $this->str[$this->last]; } - $token .= "\n"; // Adding the line ending. + $token .= "\n"; // Adding the line ending. return new Token($token, Token::TYPE_COMMENT, Token::FLAG_COMMENT_BASH); } diff --git a/libraries/sql-parser/src/Parser.php b/libraries/sql-parser/src/Parser.php index ac92aaf2b2..d6d391bd7c 100644 --- a/libraries/sql-parser/src/Parser.php +++ b/libraries/sql-parser/src/Parser.php @@ -303,8 +303,8 @@ namespace SqlParser { /** * Constructor. * - * @param mixed $list The list of tokens to be parsed. - * @param bool $strict Whether strict mode should be enabled or not. + * @param string|UtfString|TokensList $list The list of tokens to be parsed. + * @param bool $strict Whether strict mode should be enabled or not. */ public function __construct($list = null, $strict = false) { @@ -375,6 +375,7 @@ namespace SqlParser { ) { // Skipping to the end of this statement. $list->getNextOfType(Token::TYPE_DELIMITER); + $prevLastIdx = $list->idx; continue; } @@ -412,7 +413,6 @@ namespace SqlParser { } // Skipping to the end of this statement. $list->getNextOfType(Token::TYPE_DELIMITER); - // $prevLastIdx = $list->idx; continue; } diff --git a/libraries/sql-parser/src/Statements/AlterStatement.php b/libraries/sql-parser/src/Statements/AlterStatement.php index 0dfee2cff8..2c48181152 100644 --- a/libraries/sql-parser/src/Statements/AlterStatement.php +++ b/libraries/sql-parser/src/Statements/AlterStatement.php @@ -131,7 +131,7 @@ class AlterStatement extends Statement $tmp[] = $altered::build($altered); } - return 'ALTER ' . OptionsArray::build($this->options) + return 'ALTER ' . OptionsArray::build($this->options) . ' TABLE ' . Expression::build($this->table) . ' ' . implode(', ', $tmp); } diff --git a/libraries/sql-parser/src/Token.php b/libraries/sql-parser/src/Token.php index 99d239b2ba..3c36d1c375 100644 --- a/libraries/sql-parser/src/Token.php +++ b/libraries/sql-parser/src/Token.php @@ -55,7 +55,7 @@ class Token /** * Spaces, tabs, new lines, etc. - * + * * @var int */ const TYPE_WHITESPACE = 3; diff --git a/libraries/sql-parser/src/Utils/BufferedQuery.php b/libraries/sql-parser/src/Utils/BufferedQuery.php index c01ead72a3..e308fb2248 100644 --- a/libraries/sql-parser/src/Utils/BufferedQuery.php +++ b/libraries/sql-parser/src/Utils/BufferedQuery.php @@ -31,7 +31,7 @@ class BufferedQuery // Constants that describe the current status of the parser. const STATUS_STRING_SINGLE_QUOTES = 1; const STATUS_STRING_DOUBLE_QUOTES = 2; - const STATUS_STRING_BACKTICK = 3; + const STATUS_STRING_BACKTICK = 3; const STATUS_COMMENT_BASH = 4; const STATUS_COMMENT_C = 5; const STATUS_COMMENT_SQL = 6; @@ -113,7 +113,7 @@ class BufferedQuery $options ); - $this->query = ''; + $this->query = $query; $this->setDelimiter($this->options['delimiter']); } @@ -180,7 +180,7 @@ class BufferedQuery */ $loopLen = $end ? $len : $len - 16; - for (; $i < $loopLen; ++$i) { + for (; $i < $loopLen; ++$i) { /* * Handling special parses statuses. @@ -268,7 +268,8 @@ class BufferedQuery * * This optimization makes the code about 3 times faster. */ - if ((($this->query[$i] === 'D') || ($this->query[$i] === 'd')) + if (($i + 9 < $len) + && (($this->query[$i ] === 'D') || ($this->query[$i ] === 'd')) && (($this->query[$i + 1] === 'E') || ($this->query[$i + 1] === 'e')) && (($this->query[$i + 2] === 'L') || ($this->query[$i + 2] === 'l')) && (($this->query[$i + 3] === 'I') || ($this->query[$i + 3] === 'i')) @@ -277,6 +278,7 @@ class BufferedQuery && (($this->query[$i + 6] === 'T') || ($this->query[$i + 6] === 't')) && (($this->query[$i + 7] === 'E') || ($this->query[$i + 7] === 'e')) && (($this->query[$i + 8] === 'R') || ($this->query[$i + 8] === 'r')) + && (Context::isWhitespace($this->query[$i + 9])) ) { // Saving the current index to be able to revert any parsing @@ -289,13 +291,6 @@ class BufferedQuery ++$i; } - // Checking if any whitespace was found between keyword - // `DELIMITER` and the actual delimiter. - if ($iBak + 9 === $i) { - $i = $iBak; - return false; - } - // Parsing the delimiter. $delimiter = ''; while (($i < $len) && (!Context::isWhitespace($this->query[$i]))) { @@ -303,8 +298,9 @@ class BufferedQuery } // Checking if the delimiter definition ended. - if ((($i < $len) && (Context::isWhitespace($this->query[$i]))) - || (($i === $len) && ($end)) + if (($delimiter != '') + && ((($i < $len) && (Context::isWhitespace($this->query[$i]))) + || (($i === $len) && ($end))) ) { // Saving the delimiter. @@ -380,11 +376,19 @@ class BufferedQuery if (($end) && ($i === $len)) { // If the end of the buffer was reached, the buffer is emptied and // the current statement that was extracted is returned. + $ret = $this->current; + + // Emptying the buffer. $this->query = ''; $i = 0; - return trim($this->current); + + // Resetting the current statement. + $this->current = ''; + + // Returning the statement. + return trim($ret); } - return false; + return ''; } } diff --git a/libraries/sql-parser/src/Utils/Query.php b/libraries/sql-parser/src/Utils/Query.php index 4a9ea01b84..6aff11de1d 100644 --- a/libraries/sql-parser/src/Utils/Query.php +++ b/libraries/sql-parser/src/Utils/Query.php @@ -51,7 +51,7 @@ class Query * @var array */ public static $FUNCTIONS = array( - 'SUM','AVG','STD','STDDEV','MIN','MAX','BIT_OR','BIT_AND' + 'SUM', 'AVG', 'STD', 'STDDEV', 'MIN', 'MAX', 'BIT_OR', 'BIT_AND' ); /** @@ -572,10 +572,10 @@ class Query } elseif (is_string($type)) { if ($clauses[$type] > $clauseIdx) { $firstClauseIdx = $clauseIdx + 1; - $lastClauseIdx = $clauses[$type] - 1 ; + $lastClauseIdx = $clauses[$type] - 1; } else { $firstClauseIdx = $clauses[$type] + 1; - $lastClauseIdx = $clauseIdx - 1 ; + $lastClauseIdx = $clauseIdx - 1; } } diff --git a/test/classes/plugin/import/ImportSql_test.php b/test/classes/plugin/import/ImportSql_test.php index f06ae0d2d8..3703c7104d 100644 --- a/test/classes/plugin/import/ImportSql_test.php +++ b/test/classes/plugin/import/ImportSql_test.php @@ -98,7 +98,7 @@ class ImportSql_Test extends PHPUnit_Framework_TestCase //asset that all sql are executed $this->assertContains( - 'SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";', + 'SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"', $sql_query ); $this->assertContains(