gian/phpmyadmin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #11375 from udan11/import_rewrite

Browse Source 
Rewrote a part of the import mechanism.
This commit is contained in:
Marc Delisle 2015-08-11 19:16:20 -04:00
commit b619077bed
10 changed files with 69 additions and 485 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

490
libraries/plugins/import/ImportSql.class.php
View File

@ -21,103 +21,6 @@ require_once 'libraries/plugins/ImportPlugin.class.php';
*/
class ImportSql extends ImportPlugin
{
const BIG_VALUE = 2147483647;
const READ_MB_FALSE = 0;
const READ_MB_TRUE = 1;
/**
* @var string SQL delimiter
*/
private $_delimiter;
/**
* @var int SQL delimiter length
*/
private $_delimiterLength;
/**
* @var bool|int SQL delimiter position or false if not found
*/
private $_delimiterPosition = false;
/**
* @var int Query start position
*/
private $_queryBeginPosition = 0;
/**
* @var int|false First special chars position or false if not found
*/
private $_firstSearchChar = null;
/**
* @var bool Current position is in string
*/
private $_isInString = false;
/**
* @var string Quote of current string or null if out of string
*/
private $_quote = null;
/**
* @var bool Current position is in comment
*/
private $_isInComment = false;
/**
* @var string Current comment opener
*/
private $_openingComment = null;
/**
* @var bool Current position is in delimiter definition
*/
private $_isInDelimiter = false;
/**
* @var string Delimiter keyword
*/
private $_delimiterKeyword = 'DELIMITER ';
/**
* @var int Import should be done using multibytes
*/
private $_readMb = self::READ_MB_FALSE;
/**
* @var string Data to parse
*/
private $_data = null;
/**
* @var int Length of data to parse
*/
private $_dataLength = 0;
/**
* @var array List of string functions
* @todo Move this part in string functions definition file.
*/
private $_stringFunctions = array(
self::READ_MB_FALSE => array(
'substr' => 'substr',
'strlen' => 'strlen',
'strpos' => 'strpos',
'strtoupper' => 'strtoupper',
),
self::READ_MB_TRUE => array(
'substr' => 'mb_substr',
'strlen' => 'mb_strlen',
'strpos' => 'mb_strpos',
'strtoupper' => 'mb_strtoupper',
),
);
/**
* @var bool|int List of string functions to use
*/
private $_stringFctToUse = false;
/**
* Constructor
@ -189,13 +92,6 @@ class ImportSql extends ImportPlugin
);
$generalOptions->addProperty($leaf);
$leaf = new BoolPropertyItem();
$leaf->setName("read_as_multibytes");
$leaf->setText(
__('Read as multibytes')
);
$generalOptions->addProperty($leaf);
// add the main group to the root group
$importSpecificOptions->addProperty($generalOptions);
// set the options for the import plugin property item
@ -205,182 +101,7 @@ class ImportSql extends ImportPlugin
$this->properties = $importPluginProperties;
}
/**
* Look for end of string
*
* @return bool End of string found
*/
private function _searchStringEnd()
{
//Search for closing quote
$posClosingString = $this->_stringFctToUse['strpos'](
$this->_data, $this->_quote, $this->_delimiterPosition
);
if (false === $posClosingString) {
return false;
}
//Quotes escaped by quote will be considered as 2 consecutive strings
//and won't pass in this loop.
$posEscape = $posClosingString-1;
while ($this->_stringFctToUse['substr']($this->_data, $posEscape, 1) == '\\'
) {
$posEscape--;
}
// Odd count means it was escaped
$quoteEscaped = (((($posClosingString - 1) - $posEscape) % 2) === 1);
//Move after the escaped quote.
$this->_delimiterPosition = $posClosingString + 1;
if ($quoteEscaped) {
return true;
}
$this->_isInString = false;
$this->_quote = null;
return true;
}
/**
* Return the position of first SQL delimiter or false if no SQL delimiter found.
*
* @return int|bool Delimiter position or false if no delimiter found
*/
private function _findDelimiterPosition()
{
$this->_firstSearchChar = null;
$firstSqlDelimiter = null;
$matches = null;
/* while not at end of line */
while ($this->_delimiterPosition < $this->_dataLength) {
if ($this->_isInString) {
if (false === $this->_searchStringEnd()) {
return false;
}
continue;
}
if ($this->_isInComment) {
if (in_array($this->_openingComment, array('#', '-- '))) {
$posClosingComment = $this->_stringFctToUse['strpos'](
$this->_data,
"\n",
$this->_delimiterPosition
);
if (false === $posClosingComment) {
return false;
}
//Move after the end of the line.
$this->_delimiterPosition = $posClosingComment + 1;
$this->_isInComment = false;
$this->_openingComment = null;
} elseif ('/*' === $this->_openingComment) {
//Search for closing comment
$posClosingComment = $this->_stringFctToUse['strpos'](
$this->_data,
'*/',
$this->_delimiterPosition
);
if (false === $posClosingComment) {
return false;
}
//Move after closing comment.
$this->_delimiterPosition = $posClosingComment + 2;
$this->_isInComment = false;
$this->_openingComment = null;
} else {
//We shouldn't be able to come here.
//throw new Exception('Unknown case.');
break;
}
continue;
}
if ($this->_isInDelimiter) {
//Search for new line.
if (!preg_match(
"/^(.*)\n/",
$this->_stringFctToUse['substr'](
$this->_data,
$this->_delimiterPosition
),
$matches,
PREG_OFFSET_CAPTURE
)) {
return false;
}
$this->_setDelimiter($matches[1][0]);
//Start after delimiter and new line.
$this->_queryBeginPosition = $this->_delimiterPosition
+ $matches[1][1] + $this->_delimiterLength + 1;
$this->_delimiterPosition = $this->_queryBeginPosition;
$this->_isInDelimiter = false;
$firstSqlDelimiter = null;
$this->_firstSearchChar = null;
continue;
}
$matches = $this->_searchSpecialChars($matches);
$firstSqlDelimiter = $this->_searchSqlDelimiter($firstSqlDelimiter);
if (false === $firstSqlDelimiter && false === $this->_firstSearchChar) {
return false;
}
//If first char is delimiter.
if (false === $this->_firstSearchChar
|| (false !== $firstSqlDelimiter
&& $firstSqlDelimiter < $this->_firstSearchChar)
) {
$this->_delimiterPosition = $firstSqlDelimiter;
return true;
}
//Else first char is result of preg_match.
$specialChars = $matches[1][0];
//If string is opened.
if (in_array($specialChars, array('\'', '"', '`'))) {
$this->_isInString = true;
$this->_quote = $specialChars;
//Move before quote.
$this->_delimiterPosition = $this->_firstSearchChar + 1;
continue;
}
//If comment is opened.
if (in_array($specialChars, array('#', '-- ', '/*'))) {
$this->_isInComment = true;
$this->_openingComment = $specialChars;
//Move before comment opening.
$this->_delimiterPosition = $this->_firstSearchChar
+ $this->_stringFctToUse['strlen']($specialChars);
continue;
}
//If DELIMITER is found.
$specialCharsUpper = $this->_stringFctToUse['strtoupper']($specialChars);
if ($specialCharsUpper === $this->_delimiterKeyword) {
$this->_isInDelimiter = true;
$this->_delimiterPosition = $this->_firstSearchChar
+ $this->_stringFctToUse['strlen']($specialChars);
continue;
}
}
return false;
}
/**
/*
* Handles the whole import logic
*
* @param array &$sql_data 2-element array with sql data
@ -391,95 +112,63 @@ class ImportSql extends ImportPlugin
{
global $error, $timeout_passed;
//Manage multibytes or not
if (isset($_REQUEST['sql_read_as_multibytes'])) {
$this->_readMb = self::READ_MB_TRUE;
}
$this->_stringFctToUse = $this->_stringFunctions[$this->_readMb];
if (isset($_POST['sql_delimiter'])) {
$this->_setDelimiter($_POST['sql_delimiter']);
} else {
$this->_setDelimiter(';');
}
// Handle compatibility options
// Handle compatibility options.
$this->_setSQLMode($GLOBALS['dbi'], $_REQUEST);
//Initialise data.
$this->_setData(null);
$bq = new SqlParser\Utils\BufferedQuery();
if (isset($_POST['sql_delimiter'])) {
$bq->setDelimiter($_POST['sql_delimiter']);
}
/**
* will be set in PMA_importGetNextChunk()
*
* @global boolean $GLOBALS['finished']
* Will be set in PMA_importGetNextChunk().
* @global bool $GLOBALS['finished']
*/
$GLOBALS['finished'] = false;
$delimiterFound = false;
while (!$error && !$timeout_passed) {
if (false === $delimiterFound) {
while ((!$error) && (!$timeout_passed)) {
// Getting the first statement, the remaining data and the last
// delimiter.
$statement = $bq->extract();
// If there is no full statement, we are looking for more data.
if (empty($statement)) {
// Importing new data.
$newData = PMA_importGetNextChunk(200);
// Subtract data we didn't handle yet and stop processing.
if ($newData === false) {
// subtract data we didn't handle yet and stop processing
$GLOBALS['offset'] -= $this->_dataLength;
$GLOBALS['offset'] -= mb_strlen($bq->query);
break;
}
// Checking if the input buffer has finished.
if ($newData === true) {
$GLOBALS['finished'] = true;
break;
}
//Convert CR (but not CRLF) to LF otherwise all queries
//may not get executed on some platforms
$this->_addData(preg_replace("/\r($|[^\n])/", "\n$1", $newData));
unset($newData);
}
// Convert CR (but not CRLF) to LF otherwise all queries may
// not get executed on some platforms.
$bq->query .= preg_replace("/\r($|[^\n])/", "\n$1", $newData);
//Find quotes, comments, delimiter definition or delimiter itself.
$delimiterFound = $this->_findDelimiterPosition();
//If no delimiter found, restart and get more data.
if (false === $delimiterFound) {
continue;
}
PMA_importRunQuery(
$this->_stringFctToUse['substr'](
$this->_data,
$this->_queryBeginPosition,
$this->_delimiterPosition - $this->_queryBeginPosition
), //Query to execute
$this->_stringFctToUse['substr'](
$this->_data,
0,
$this->_delimiterPosition + $this->_delimiterLength
), //Query to display
false,
$sql_data
);
$this->_setData(
$this->_stringFctToUse['substr'](
$this->_data,
$this->_delimiterPosition + $this->_delimiterLength
)
);
// Executing the query.
PMA_importRunQuery($statement, $statement, false, $sql_data);
}
if (! $timeout_passed) {
//Commit any possible data in buffers
PMA_importRunQuery(
$this->_stringFctToUse['substr'](
$this->_data,
$this->_queryBeginPosition
), //Query to execute
$this->_data,
false,
$sql_data
);
// Extracting remaining statements.
while ((!$error) && (!$timeout_passed)
&& ($statement = $bq->extract(true))
) {
PMA_importRunQuery($statement, $statement, false, $sql_data);
}
// Finishing.
PMA_importRunQuery('', '', false, $sql_data);
}
@ -508,113 +197,4 @@ class ImportSql extends ImportPlugin
);
}
}
/**
* Look for special chars: comment, string or DELIMITER
*
* @param array $matches Special chars found in data
*
* @return array matches
*/
private function _searchSpecialChars(
$matches
) {
//Don't look for a string/comment/"DELIMITER" if not found previously
//or if it's still after current position.
if (null === $this->_firstSearchChar
|| (false !== $this->_firstSearchChar
&& $this->_firstSearchChar < $this->_delimiterPosition)
) {
$bFind = preg_match(
'/(\'|"|#|-- |\/\*|`|(?i)(?<![A-Z0-9_])'
. $this->_delimiterKeyword . ')/',
$this->_stringFctToUse['substr'](
$this->_data,
$this->_delimiterPosition
),
$matches,
PREG_OFFSET_CAPTURE
);
if (1 === $bFind) {
$this->_firstSearchChar = $matches[1][1] + $this->_delimiterPosition;
} else {
$this->_firstSearchChar = false;
}
}
return $matches;
}
/**
* Look for SQL delimiter
*
* @param int $firstSqlDelimiter First found char position
*
* @return int
*/
private function _searchSqlDelimiter($firstSqlDelimiter)
{
//Don't look for the SQL delimiter if not found previously
//or if it's still after current position.
if (null === $firstSqlDelimiter
|| (false !== $firstSqlDelimiter
&& $firstSqlDelimiter < $this->_delimiterPosition)
) {
// the cost of doing this one with preg_match() would be too high
$firstSqlDelimiter = $this->_stringFctToUse['strpos'](
$this->_data,
$this->_delimiter,
$this->_delimiterPosition
);
}
return $firstSqlDelimiter;
}
/**
* Set new delimiter
*
* @param string $delimiter New delimiter
*
* @return int delimiter length
*/
private function _setDelimiter($delimiter)
{
$this->_delimiter = $delimiter;
$this->_delimiterLength = $this->_stringFctToUse['strlen']($delimiter);
return $this->_delimiterLength;
}
/**
* Set data to parse
*
* @param string $data Data to parse
*
* @return int Data length
*/
private function _setData($data)
{
$this->_data = ltrim($data);
$this->_dataLength = $this->_stringFctToUse['strlen']($this->_data);
$this->_queryBeginPosition = 0;
$this->_delimiterPosition = 0;
return $this->_dataLength;
}
/**
* Add data to parse
*
* @param string $data Data to add to data to parse
*
* @return int Data length
*/
private function _addData($data)
{
$this->_data .= $data;
$this->_dataLength += $this->_stringFctToUse['strlen']($data);
return $this->_dataLength;
}
}
}

2
libraries/sql-parser/src/Components/JoinKeyword.php
View File

@ -71,7 +71,7 @@ class JoinKeyword extends Component
{
$ret = array();
$expr = new JoinKeyword();;
$expr = new JoinKeyword();
/**
* The state of the parser.

8
libraries/sql-parser/src/Components/OptionsArray.php
View File

@ -55,10 +55,10 @@ class OptionsArray extends Component
{
$ret = new OptionsArray();
/**
* The ID that will be assigned to duplicate options.
* @var int $lastAssignedId
*/
/**
* The ID that will be assigned to duplicate options.
* @var int $lastAssignedId
*/
$lastAssignedId = count($options) + 1;
/**

2
libraries/sql-parser/src/Lexer.php
View File

@ -506,7 +506,7 @@ namespace SqlParser {
while ((++$this->last < $this->len) && ($this->str[$this->last] !== "\n")) {
$token .= $this->str[$this->last];
}
$token .= "\n"; // Adding the line ending.
$token .= "\n"; // Adding the line ending.
return new Token($token, Token::TYPE_COMMENT, Token::FLAG_COMMENT_BASH);
}

6
libraries/sql-parser/src/Parser.php
View File

@ -303,8 +303,8 @@ namespace SqlParser {
/**
* Constructor.
*
* @param mixed $list The list of tokens to be parsed.
* @param bool $strict Whether strict mode should be enabled or not.
* @param string|UtfString|TokensList $list The list of tokens to be parsed.
* @param bool $strict Whether strict mode should be enabled or not.
*/
public function __construct($list = null, $strict = false)
{
@ -375,6 +375,7 @@ namespace SqlParser {
) {
// Skipping to the end of this statement.
$list->getNextOfType(Token::TYPE_DELIMITER);
$prevLastIdx = $list->idx;
continue;
}
@ -412,7 +413,6 @@ namespace SqlParser {
}
// Skipping to the end of this statement.
$list->getNextOfType(Token::TYPE_DELIMITER);
//
$prevLastIdx = $list->idx;
continue;
}

2
libraries/sql-parser/src/Statements/AlterStatement.php
View File

@ -131,7 +131,7 @@ class AlterStatement extends Statement
$tmp[] = $altered::build($altered);
}
return 'ALTER ' . OptionsArray::build($this->options)
return 'ALTER ' . OptionsArray::build($this->options)
. ' TABLE ' . Expression::build($this->table)
. ' ' . implode(', ', $tmp);
}

2
libraries/sql-parser/src/Token.php
View File

@ -55,7 +55,7 @@ class Token
/**
* Spaces, tabs, new lines, etc.
*
*
* @var int
*/
const TYPE_WHITESPACE = 3;

34
libraries/sql-parser/src/Utils/BufferedQuery.php
View File

@ -31,7 +31,7 @@ class BufferedQuery
// Constants that describe the current status of the parser.
const STATUS_STRING_SINGLE_QUOTES = 1;
const STATUS_STRING_DOUBLE_QUOTES = 2;
const STATUS_STRING_BACKTICK = 3;
const STATUS_STRING_BACKTICK = 3;
const STATUS_COMMENT_BASH = 4;
const STATUS_COMMENT_C = 5;
const STATUS_COMMENT_SQL = 6;
@ -113,7 +113,7 @@ class BufferedQuery
$options
);
$this->query = '';
$this->query = $query;
$this->setDelimiter($this->options['delimiter']);
}
@ -180,7 +180,7 @@ class BufferedQuery
*/
$loopLen = $end ? $len : $len - 16;
for (; $i < $loopLen; ++$i) {
for (; $i < $loopLen; ++$i) {
/*
* Handling special parses statuses.
@ -268,7 +268,8 @@ class BufferedQuery
*
* This optimization makes the code about 3 times faster.
*/
if ((($this->query[$i] === 'D') || ($this->query[$i] === 'd'))
if (($i + 9 < $len)
&& (($this->query[$i ] === 'D') || ($this->query[$i ] === 'd'))
&& (($this->query[$i + 1] === 'E') || ($this->query[$i + 1] === 'e'))
&& (($this->query[$i + 2] === 'L') || ($this->query[$i + 2] === 'l'))
&& (($this->query[$i + 3] === 'I') || ($this->query[$i + 3] === 'i'))
@ -277,6 +278,7 @@ class BufferedQuery
&& (($this->query[$i + 6] === 'T') || ($this->query[$i + 6] === 't'))
&& (($this->query[$i + 7] === 'E') || ($this->query[$i + 7] === 'e'))
&& (($this->query[$i + 8] === 'R') || ($this->query[$i + 8] === 'r'))
&& (Context::isWhitespace($this->query[$i + 9]))
) {
// Saving the current index to be able to revert any parsing
@ -289,13 +291,6 @@ class BufferedQuery
++$i;
}
// Checking if any whitespace was found between keyword
// `DELIMITER` and the actual delimiter.
if ($iBak + 9 === $i) {
$i = $iBak;
return false;
}
// Parsing the delimiter.
$delimiter = '';
while (($i < $len) && (!Context::isWhitespace($this->query[$i]))) {
@ -303,8 +298,9 @@ class BufferedQuery
}
// Checking if the delimiter definition ended.
if ((($i < $len) && (Context::isWhitespace($this->query[$i])))
|| (($i === $len) && ($end))
if (($delimiter != '')
&& ((($i < $len) && (Context::isWhitespace($this->query[$i])))
|| (($i === $len) && ($end)))
) {
// Saving the delimiter.
@ -380,11 +376,19 @@ class BufferedQuery
if (($end) && ($i === $len)) {
// If the end of the buffer was reached, the buffer is emptied and
// the current statement that was extracted is returned.
$ret = $this->current;
// Emptying the buffer.
$this->query = '';
$i = 0;
return trim($this->current);
// Resetting the current statement.
$this->current = '';
// Returning the statement.
return trim($ret);
}
return false;
return '';
}
}

6
libraries/sql-parser/src/Utils/Query.php
View File

@ -51,7 +51,7 @@ class Query
* @var array
*/
public static $FUNCTIONS = array(
'SUM','AVG','STD','STDDEV','MIN','MAX','BIT_OR','BIT_AND'
'SUM', 'AVG', 'STD', 'STDDEV', 'MIN', 'MAX', 'BIT_OR', 'BIT_AND'
);
/**
@ -572,10 +572,10 @@ class Query
} elseif (is_string($type)) {
if ($clauses[$type] > $clauseIdx) {
$firstClauseIdx = $clauseIdx + 1;
$lastClauseIdx = $clauses[$type] - 1 ;
$lastClauseIdx = $clauses[$type] - 1;
} else {
$firstClauseIdx = $clauses[$type] + 1;
$lastClauseIdx = $clauseIdx - 1 ;
$lastClauseIdx = $clauseIdx - 1;
}
}

2
test/classes/plugin/import/ImportSql_test.php
View File

@ -98,7 +98,7 @@ class ImportSql_Test extends PHPUnit_Framework_TestCase
//asset that all sql are executed
$this->assertContains(
'SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";',
'SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"',
$sql_query
);
$this->assertContains(