diff --git a/docs/import_export.rst b/docs/import_export.rst index b0144ebf42..38f0dde3f8 100644 --- a/docs/import_export.rst +++ b/docs/import_export.rst @@ -264,6 +264,13 @@ rendered, for example in following document: \end{document} +TOON +---- + +Data can be exported to TOON `(Token-Oriented Object Notation) `_, +which is a text-based data format. It encodes the JSON data model in a compact way. +TOON is designed to reduce token usage in LLM prompts while staying human-readable. + MediaWiki --------- diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index c178b7636f..b476c4eb58 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -783,13 +783,13 @@ parameters: - message: '#^Cannot cast mixed to int\.$#' identifier: cast.int - count: 2 + count: 3 path: src/Config/Settings/Export.php - message: '#^Cannot cast mixed to string\.$#' identifier: cast.string - count: 23 + count: 24 path: src/Config/Settings/Export.php - @@ -5812,7 +5812,7 @@ parameters: path: src/Export/Options.php - - message: '#^Only booleans are allowed in &&, bool\|int\<0, max\>\|string given on the right side\.$#' + message: '#^Only booleans are allowed in &&, bool\|int\|string given on the right side\.$#' identifier: booleanAnd.rightNotBoolean count: 1 path: src/Export/Options.php @@ -8901,6 +8901,21 @@ parameters: count: 2 path: src/Plugins/Export/ExportTexytext.php + - + message: ''' + #^Call to deprecated method getInstance\(\) of class PhpMyAdmin\\Dbal\\DatabaseInterface\: + Use dependency injection instead\.$# + ''' + identifier: staticMethod.deprecated + count: 2 + path: src/Plugins/Export/ExportToon.php + + - + message: '#^Cannot cast mixed to int\.$#' + identifier: cast.int + count: 1 + path: src/Plugins/Export/ExportToon.php + - message: '#^Construct empty\(\) is not allowed\. Use more strict comparison\.$#' identifier: empty.notAllowed diff --git a/psalm-baseline.xml b/psalm-baseline.xml index 997e775b14..95724f5f33 100644 --- a/psalm-baseline.xml +++ b/psalm-baseline.xml @@ -5997,6 +5997,20 @@ + + + + + + + toon_indent]]> + toon_separator]]> + + + indent]]> + separator]]> + + diff --git a/src/Config/Settings/Export.php b/src/Config/Settings/Export.php index e5ac8c0e7d..7fdd635a27 100644 --- a/src/Config/Settings/Export.php +++ b/src/Config/Settings/Export.php @@ -55,6 +55,9 @@ use function in_array; * csv_escaped: string, * csv_terminated: string, * csv_removeCRLF: bool, + * toon_indent: int, + * toon_separator: string, + * toon_structure_or_data: StructureOrDataType, * excel_columns: bool, * excel_null: string, * excel_edition: 'mac_excel2003'|'mac_excel2008'|'win', @@ -442,6 +445,29 @@ final class Export */ public bool $csv_removeCRLF; + /** + * ```php + * $cfg['Export']['toon_indent'] = 2; + * ``` + */ + public int $toon_indent; + + /** + * ```php + * $cfg['Export']['toon_separator'] = ','; + * ``` + */ + public string $toon_separator; + + /** + * ```php + * $cfg['Export']['toon_structure_or_data'] = 'structure_and_data'; + * ``` + * + * @psalm-var StructureOrDataType + */ + public string $toon_structure_or_data; + /** * ```php * $cfg['Export']['excel_columns'] = true; @@ -1031,6 +1057,9 @@ final class Export $this->csv_escaped = $this->setCsvEscaped($export); $this->csv_terminated = $this->setCsvTerminated($export); $this->csv_removeCRLF = $this->setCsvRemoveCRLF($export); + $this->toon_indent = $this->setToonIndent($export); + $this->toon_separator = $this->setToonSeparator($export); + $this->toon_structure_or_data = $this->setToonStructureOrData($export); $this->excel_columns = $this->setExcelColumns($export); $this->excel_null = $this->setExcelNull($export); $this->excel_edition = $this->setExcelEdition($export); @@ -1146,6 +1175,9 @@ final class Export 'csv_escaped' => $this->csv_escaped, 'csv_terminated' => $this->csv_terminated, 'csv_removeCRLF' => $this->csv_removeCRLF, + 'toon_indent' => $this->toon_indent, + 'toon_separator' => $this->toon_separator, + 'toon_structure_or_data' => $this->toon_structure_or_data, 'excel_columns' => $this->excel_columns, 'excel_null' => $this->excel_null, 'excel_edition' => $this->excel_edition, @@ -1680,6 +1712,43 @@ final class Export return (bool) $export['csv_removeCRLF']; } + /** @param array $export */ + private function setToonIndent(array $export): int + { + if (! isset($export['toon_indent'])) { + return 2; + } + + return (int) $export['toon_indent']; + } + + /** @param array $export */ + private function setToonSeparator(array $export): string + { + if (! isset($export['toon_separator'])) { + return ','; + } + + return (string) $export['toon_separator']; + } + + /** + * @param array $export + * + * @psalm-return StructureOrDataType + */ + private function setToonStructureOrData(array $export): string + { + if ( + ! isset($export['toon_structure_or_data']) + || ! in_array($export['toon_structure_or_data'], ['structure', 'data'], true) + ) { + return 'structure_and_data'; + } + + return $export['toon_structure_or_data']; + } + /** @param array $export */ private function setExcelColumns(array $export): bool { diff --git a/src/Plugins/Export/ExportToon.php b/src/Plugins/Export/ExportToon.php new file mode 100644 index 0000000000..9caa8436c8 --- /dev/null +++ b/src/Plugins/Export/ExportToon.php @@ -0,0 +1,228 @@ +setText('TOON'); + $exportPluginProperties->setExtension('toon'); + $exportPluginProperties->setMimeType('text/toon'); + $exportPluginProperties->setForceFile(true); + $exportPluginProperties->setOptionsText(__('Options')); + + // create the root group that will be the options field for + // $exportPluginProperties + // this will be shown as "Format specific options" + $exportSpecificOptions = new OptionsPropertyRootGroup('Format Specific Options'); + + // general options main group + $generalOptions = new OptionsPropertyMainGroup('general_opts'); + // create leaf items and add them to the group + $leaf = new TextPropertyItem( + 'separator', + __('Columns separated with:'), + ); + $generalOptions->addProperty($leaf); + $leaf = new TextPropertyItem( + 'indent', + __('Indentation:'), + ); + $generalOptions->addProperty($leaf); + // create primary items and add them to the group + $leaf = new HiddenPropertyItem('structure_or_data'); + $generalOptions->addProperty($leaf); + // add the main group to the root group + $exportSpecificOptions->addProperty($generalOptions); + + // set the options for the export plugin property item + $exportPluginProperties->setOptions($exportSpecificOptions); + + return $exportPluginProperties; + } + + /** + * Outputs the content of a table in JSON format + * + * @param string $db database name + * @param string $table table name + * @param string $sqlQuery SQL query for obtaining data + * @param mixed[] $aliases Aliases of db/table/columns + */ + public function exportData( + string $db, + string $table, + string $sqlQuery, + array $aliases = [], + ): void { + $dbAlias = $this->getDbAlias($aliases, $db); + $tableAlias = $this->getTableAlias($aliases, $db, $table); + $dbi = DatabaseInterface::getInstance(); + $result = $dbi->query($sqlQuery); + + $columnsCnt = $result->numFields(); + $rowsCnt = $result->numRows(); + $fieldsMeta = $dbi->getFieldsMeta($result); + + $columns = []; + foreach ($fieldsMeta as $i => $field) { + $colAs = $this->getColumnAlias($aliases, $db, $table, $field->name); + + $columns[$i] = $colAs; + } + + $buffer = sprintf( + '%s.%s[%s%s]{', + $dbAlias, + $tableAlias, + $rowsCnt, + $this->separator !== ',' ? $this->separator : '', + ); + foreach ($columns as $index => $column) { + $buffer .= $column; + + // phpcs:ignore SlevomatCodingStandard.ControlStructures.EarlyExit.EarlyExitNotUsed + if ($index !== array_key_last($columns)) { + $buffer .= $this->separator; + } + } + + $buffer .= "}:\n"; + + $this->outputHandler->addLine($buffer); + + $insertedLines = 0; + while ($row = $result->fetchRow()) { + $buffer = ''; + + foreach ($row as $index => $col) { + if ($index === 0) { + $buffer .= str_repeat(' ', $this->indent); + } + + if ($col === null) { + $buffer .= 'null'; + continue; + } + + $buffer .= $col; + + // phpcs:ignore SlevomatCodingStandard.ControlStructures.EarlyExit.EarlyExitNotUsed + if ($index !== $columnsCnt - 1) { + $buffer .= $this->separator; + } + } + + $insertedLines++; + $buffer .= $insertedLines === $rowsCnt ? "\n\n" : "\n"; + $this->outputHandler->addLine($buffer); + } + } + + /** + * Outputs result raw query in TOON format + * + * @param string $db the database where the query is executed + * @param string $sqlQuery the rawquery to output + */ + public function exportRawQuery(string $db, string $sqlQuery): void + { + if ($db !== '') { + DatabaseInterface::getInstance()->selectDb($db); + } + + $this->exportData($db, '', $sqlQuery); + } + + private function setupExportConfiguration(): void + { + $this->separator = str_replace('\\t', "\011", $this->separator); + } + + public function setExportOptions(ServerRequest $request, Export $exportConfig): void + { + // phpcs:disable Squiz.NamingConventions.ValidVariableName.MemberNotCamelCaps + $this->structureOrData = $this->setStructureOrData( + $request->getParsedBodyParam('toon_structure_or_data'), + $exportConfig->toon_structure_or_data, + StructureOrData::StructureAndData, + ); + + $this->separator = $this->setStringValue( + $request->getParsedBodyParam('toon_separator'), + $exportConfig->toon_separator ?? $this->separator, + ); + + $this->indent = $this->setIntValue( + (int) $request->getParsedBodyParam('toon_indent'), + $exportConfig->toon_indent ?? $this->indent, + ); + // phpcs:enable Squiz.NamingConventions.ValidVariableName.MemberNotCamelCaps + + $this->setupExportConfiguration(); + } + + private function setStringValue(mixed $fromRequest, mixed $fromConfig): string + { + if (is_string($fromRequest) && $fromRequest !== '') { + return $fromRequest; + } + + if (is_string($fromConfig) && $fromConfig !== '') { + return $fromConfig; + } + + return ''; + } + + private function setIntValue(mixed $fromRequest, mixed $fromConfig): int + { + if (is_int($fromRequest)) { + return $fromRequest; + } + + if (is_int($fromConfig)) { + return $fromConfig; + } + + return 0; + } +} diff --git a/tests/unit/Plugins/Export/ExportToonTest.php b/tests/unit/Plugins/Export/ExportToonTest.php new file mode 100644 index 0000000000..9d91682306 --- /dev/null +++ b/tests/unit/Plugins/Export/ExportToonTest.php @@ -0,0 +1,229 @@ +getExportToon(); + + $method = new ReflectionMethod(ExportToon::class, 'setProperties'); + $method->invoke($exportToon, null); + + $attrProperties = new ReflectionProperty(ExportToon::class, 'properties'); + $properties = $attrProperties->getValue($exportToon); + + self::assertInstanceOf(ExportPluginProperties::class, $properties); + + self::assertSame( + 'TOON', + $properties->getText(), + ); + + self::assertSame( + 'toon', + $properties->getExtension(), + ); + + self::assertSame( + 'text/toon', + $properties->getMimeType(), + ); + + self::assertSame( + 'Options', + $properties->getOptionsText(), + ); + + $options = $properties->getOptions(); + + self::assertInstanceOf(OptionsPropertyRootGroup::class, $options); + + self::assertSame( + 'Format Specific Options', + $options->getName(), + ); + + $generalOptionsArray = $options->getProperties(); + $generalOptions = $generalOptionsArray->current(); + + self::assertInstanceOf(OptionsPropertyMainGroup::class, $generalOptions); + + self::assertSame( + 'general_opts', + $generalOptions->getName(), + ); + + $generalProperties = $generalOptions->getProperties(); + + $property = $generalProperties->current(); + $generalProperties->next(); + + self::assertInstanceOf(TextPropertyItem::class, $property); + + self::assertSame( + 'separator', + $property->getName(), + ); + + self::assertSame( + 'Columns separated with:', + $property->getText(), + ); + + $property = $generalProperties->current(); + $generalProperties->next(); + + self::assertInstanceOf(TextPropertyItem::class, $property); + + self::assertSame( + 'indent', + $property->getName(), + ); + + self::assertSame( + 'Indentation:', + $property->getText(), + ); + + $property = $generalProperties->current(); + + self::assertInstanceOf(HiddenPropertyItem::class, $property); + + self::assertSame( + 'structure_or_data', + $property->getName(), + ); + } + + public function testExportHeader(): void + { + $exportToon = $this->getExportToon(); + $this->expectNotToPerformAssertions(); + $exportToon->exportHeader(); + } + + public function testExportFooter(): void + { + $exportToon = $this->getExportToon(); + $this->expectNotToPerformAssertions(); + $exportToon->exportFooter(); + } + + public function testExportDBHeader(): void + { + $exportToon = $this->getExportToon(); + $this->expectNotToPerformAssertions(); + $exportToon->exportDBHeader('testDB'); + } + + public function testExportDBFooter(): void + { + $exportToon = $this->getExportToon(); + $this->expectNotToPerformAssertions(); + $exportToon->exportDBFooter('testDB'); + } + + public function testExportDBCreate(): void + { + $exportToon = $this->getExportToon(); + $this->expectNotToPerformAssertions(); + $exportToon->exportDBCreate('testDB'); + } + + public function testExportData(): void + { + $exportToon = $this->getExportToon(); + + $request = ServerRequestFactory::create()->createServerRequest('POST', 'https://example.com/') + ->withParsedBody([]); + + $exportToon->setExportOptions($request, new Export()); + + ob_start(); + $exportToon->exportData('test_db', 'test_table', 'SELECT * FROM `test_db`.`test_table`;'); + $result = ob_get_clean(); + + self::assertIsString($result); + self::assertSame( + 'test_db.test_table[3]{id,name,datetimefield}:' . "\n" . + ' 1,abcd,2011-01-20 02:00:02' . "\n" . + ' 2,foo,2010-01-20 02:00:02' . "\n" . + ' 3,Abcd,2012-01-20 02:00:02' . "\n\n", + $result, + ); + } + + public function testExportDataWithCustomConfig(): void + { + $exportToon = $this->getExportToon(); + $request = ServerRequestFactory::create()->createServerRequest('POST', 'https://example.com/') + ->withParsedBody(['toon_separator' => '|', 'toon_indent' => '4']); + + $exportToon->setExportOptions($request, new Export()); + + ob_start(); + $exportToon->exportData('test_db', 'test_table', 'SELECT * FROM `test_db`.`test_table`;'); + $result = ob_get_clean(); + + self::assertIsString($result); + self::assertSame( + 'test_db.test_table[3|]{id|name|datetimefield}:' . "\n" . + ' 1|abcd|2011-01-20 02:00:02' . "\n" . + ' 2|foo|2010-01-20 02:00:02' . "\n" . + ' 3|Abcd|2012-01-20 02:00:02' . "\n\n", + $result, + ); + } + + private function getExportToon(DatabaseInterface|null $dbi = null): ExportToon + { + $dbi ??= $this->createDatabaseInterface(); + $config = new Config(); + $relation = new Relation($dbi, $config); + + return new ExportToon($relation, new OutputHandler(), new Transformations($dbi, $relation), $dbi, $config); + } +} diff --git a/tests/unit/PluginsTest.php b/tests/unit/PluginsTest.php index fb13304a3d..24212b307f 100644 --- a/tests/unit/PluginsTest.php +++ b/tests/unit/PluginsTest.php @@ -46,7 +46,7 @@ class PluginsTest extends AbstractTestCase $isCurl = extension_loaded('curl'); self::assertSame(ExportType::Database, ExportPlugin::$exportType); self::assertFalse(ExportPlugin::$singleTable); - $pluginCount = $isCurl ? 14 : 13; + $pluginCount = $isCurl ? 15 : 14; self::assertCount($pluginCount, $plugins); self::assertContainsOnlyInstancesOf(ExportPlugin::class, $plugins); } @@ -432,6 +432,12 @@ class PluginsTest extends AbstractTestCase +

TOON

+
  • +
  • +
+
+

YAML

This format has no options