From 65ee6dc8e82a928010e6d9a8894217cefdd4b70e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Andruszkiewicz?= Date: Thu, 22 May 2025 17:55:07 +0200 Subject: [PATCH 1/2] BUG#37800574 Request for adding a flag in shell log to identify any bad records during load operation When executing util.importTable() or util.loadDump(), if LOAD DATA statement finished with warnings, Shell would print on screen/write to the log file messages like: Warning: schema@table@123.tsv.zst error 1062: Duplicate entry '1234567' for key 'table.PRIMARY' It was difficult to locate these messages in the log file, as they lacked some common prefix that would allow to i.e. easily grep for them. The message has been improved: Warning: An error has been reported while loading data into `schema`.`table` from 'schema@table@123.tsv.zst' file, error 1062: Duplicate entry '1234567' for key 'table.PRIMARY' Change-Id: I0ffc43e9af45fb14e92fd3c9a5714c4622da3e51 --- modules/util/import_table/load_data.cc | 90 ++++++++++--------- .../scripts/util_import_table_norecord.js | 30 +++---- .../scripts/util_import_table_norecord.py | 20 ++--- 3 files changed, 73 insertions(+), 67 deletions(-) diff --git a/modules/util/import_table/load_data.cc b/modules/util/import_table/load_data.cc index d1e010c55d..f3b4c94182 100644 --- a/modules/util/import_table/load_data.cc +++ b/modules/util/import_table/load_data.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2024, Oracle and/or its affiliates. + * Copyright (c) 2018, 2025, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, version 2.0, @@ -800,7 +800,7 @@ void Load_data_worker::execute( throw std::exception(e); } - const std::string task = fi.filehandler->filename(); + const auto filename = fi.filehandler->filename(); std::shared_ptr load_result = nullptr; const std::string query_prefix = shcore::sqlformat( "LOAD DATA LOCAL INFILE ? ", fi.filehandler->full_path().masked()); @@ -862,7 +862,8 @@ void Load_data_worker::execute( { const char *mysql_info = session->get_mysql_info(); const auto status = - worker_name + task + ": " + (mysql_info ? mysql_info : "ERROR") + + worker_name + filename + ": " + + (mysql_info ? mysql_info : "ERROR") + ((options.max_trx_size == 0 && max_trx_size == 0) ? "" : (fi.continuation @@ -892,49 +893,54 @@ void Load_data_worker::execute( } if (warnings_num > 0) { - // show first k warnings, where k = warnings_to_show - constexpr int warnings_to_show = 5; - auto w = load_result->fetch_one_warning(); - - for (int i = 0; w && i < warnings_to_show; - w = load_result->fetch_one_warning(), i++) { - const std::string msg = - task + " error " + std::to_string(w->code) + ": " + w->msg; - - switch (w->level) { - case mysqlshdk::db::Warning::Level::Error: - mysqlsh::current_console()->print_error(msg); - break; - case mysqlshdk::db::Warning::Level::Warn: - mysqlsh::current_console()->print_warning(msg); - break; - case mysqlshdk::db::Warning::Level::Note: - mysqlsh::current_console()->print_note(msg); - break; + // number of warnings to show + constexpr std::size_t k_warnings_to_show = 5; + std::size_t warnings_count = 0; + + while (const auto w = load_result->fetch_one_warning()) { + const std::string msg = shcore::str_format( + "LOAD DATA error for `%s`.`%s` in file '%s', error %" PRIu32 + ": %s", + m_opt.schema().c_str(), m_opt.table().c_str(), filename.c_str(), + w->code, w->msg.c_str()); + + if (warnings_count++ < k_warnings_to_show) { + // show first k warnings + switch (w->level) { + case mysqlshdk::db::Warning::Level::Error: + mysqlsh::current_console()->print_error(msg); + break; + + case mysqlshdk::db::Warning::Level::Warn: + mysqlsh::current_console()->print_warning(msg); + break; + + case mysqlshdk::db::Warning::Level::Note: + mysqlsh::current_console()->print_note(msg); + break; + } + } else { + // log remaining warnings + switch (w->level) { + case mysqlshdk::db::Warning::Level::Error: + log_error("%s", msg.c_str()); + break; + + case mysqlshdk::db::Warning::Level::Warn: + log_warning("%s", msg.c_str()); + break; + + case mysqlshdk::db::Warning::Level::Note: + log_info("%s", msg.c_str()); + break; + } } } - // log remaining warnings - size_t remaining_warnings_count = 0; - for (; w; w = load_result->fetch_one_warning()) { - remaining_warnings_count++; - const std::string msg = - task + " error " + std::to_string(w->code) + ": " + w->msg; - - switch (w->level) { - case mysqlshdk::db::Warning::Level::Error: - log_error("%s", msg.c_str()); - break; - case mysqlshdk::db::Warning::Level::Warn: - log_warning("%s", msg.c_str()); - break; - case mysqlshdk::db::Warning::Level::Note: - log_info("%s", msg.c_str()); - break; - } - } + if (warnings_count > k_warnings_to_show) { + const auto remaining_warnings_count = + warnings_count - k_warnings_to_show; - if (remaining_warnings_count > 0) { mysqlsh::current_console()->print_info( "Check mysqlsh.log for " + std::to_string(remaining_warnings_count) + " more warning" + diff --git a/unittest/scripts/auto/js_shell/scripts/util_import_table_norecord.js b/unittest/scripts/auto/js_shell/scripts/util_import_table_norecord.js index e9bbff4e36..7e80e223ce 100644 --- a/unittest/scripts/auto/js_shell/scripts/util_import_table_norecord.js +++ b/unittest/scripts/auto/js_shell/scripts/util_import_table_norecord.js @@ -114,11 +114,11 @@ util.importTable(__import_data_path + '/world_x_cities.csv', { var keyname = testutil.versionCheck(__version, '<', '8.0.19') ? `'PRIMARY'` : `'cities.PRIMARY'`; -EXPECT_STDOUT_CONTAINS("WARNING: world_x_cities.csv error 1062: Duplicate entry '1' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: world_x_cities.csv error 1062: Duplicate entry '2' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: world_x_cities.csv error 1062: Duplicate entry '3' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: world_x_cities.csv error 1062: Duplicate entry '4' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: world_x_cities.csv error 1062: Duplicate entry '5' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities.csv', error 1062: Duplicate entry '1' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities.csv', error 1062: Duplicate entry '2' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities.csv', error 1062: Duplicate entry '3' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities.csv', error 1062: Duplicate entry '4' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities.csv', error 1062: Duplicate entry '5' for key " + keyname); EXPECT_STDOUT_CONTAINS("File '" + filename_for_output(__import_data_path + "/world_x_cities.csv") + "' (250.53 KB) was imported in "); EXPECT_STDOUT_CONTAINS("Total rows affected in " + target_schema + ".cities: Records: 4079 Deleted: 0 Skipped: 4079 Warnings: 4079"); @@ -129,11 +129,11 @@ util.importTable(__import_data_path + '/world_x_cities_header.csv', { }); EXPECT_STDOUT_CONTAINS("wl12193.cities: Records: 4079 Deleted: 0 Skipped: 4079 Warnings: 4079"); -EXPECT_STDOUT_CONTAINS("WARNING: world_x_cities_header.csv error 1062: Duplicate entry '1' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: world_x_cities_header.csv error 1062: Duplicate entry '2' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: world_x_cities_header.csv error 1062: Duplicate entry '3' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: world_x_cities_header.csv error 1062: Duplicate entry '4' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: world_x_cities_header.csv error 1062: Duplicate entry '5' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities_header.csv', error 1062: Duplicate entry '1' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities_header.csv', error 1062: Duplicate entry '2' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities_header.csv', error 1062: Duplicate entry '3' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities_header.csv', error 1062: Duplicate entry '4' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities_header.csv', error 1062: Duplicate entry '5' for key " + keyname); EXPECT_STDOUT_CONTAINS("File '" + filename_for_output(__import_data_path + "/world_x_cities_header.csv") + "' (250.53 KB) was imported in "); EXPECT_STDOUT_CONTAINS("Total rows affected in " + target_schema + ".cities: Records: 4079 Deleted: 0 Skipped: 4079 Warnings: 4079"); @@ -202,11 +202,11 @@ var rc = callMysqlsh([uri, keyname = testutil.versionCheck(__version, '<', '8.0.19') ? `'PRIMARY'` : `'document_store.PRIMARY'`; EXPECT_STDOUT_CONTAINS("wl12193.document_store: Records: 25359 Deleted: 0 Skipped: 25359 Warnings: 25359"); -EXPECT_STDOUT_CONTAINS("WARNING: primer-dataset-id.json error 1062: Duplicate entry '000000000001' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: primer-dataset-id.json error 1062: Duplicate entry '000000000002' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: primer-dataset-id.json error 1062: Duplicate entry '000000000003' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: primer-dataset-id.json error 1062: Duplicate entry '000000000004' for key " + keyname); -EXPECT_STDOUT_CONTAINS("WARNING: primer-dataset-id.json error 1062: Duplicate entry '000000000005' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`document_store` in file 'primer-dataset-id.json', error 1062: Duplicate entry '000000000001' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`document_store` in file 'primer-dataset-id.json', error 1062: Duplicate entry '000000000002' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`document_store` in file 'primer-dataset-id.json', error 1062: Duplicate entry '000000000003' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`document_store` in file 'primer-dataset-id.json', error 1062: Duplicate entry '000000000004' for key " + keyname); +EXPECT_STDOUT_CONTAINS("WARNING: LOAD DATA error for `wl12193`.`document_store` in file 'primer-dataset-id.json', error 1062: Duplicate entry '000000000005' for key " + keyname); EXPECT_STDOUT_CONTAINS("File '" + filename_for_output(__import_data_path + "/primer-dataset-id.json") + "' (11.29 MB) was imported in "); EXPECT_STDOUT_CONTAINS("Total rows affected in " + target_schema + ".document_store: Records: 25359 Deleted: 0 Skipped: 25359 Warnings: 25359"); diff --git a/unittest/scripts/auto/py_shell/scripts/util_import_table_norecord.py b/unittest/scripts/auto/py_shell/scripts/util_import_table_norecord.py index ac27a69b37..dabccdc487 100644 --- a/unittest/scripts/auto/py_shell/scripts/util_import_table_norecord.py +++ b/unittest/scripts/auto/py_shell/scripts/util_import_table_norecord.py @@ -120,11 +120,11 @@ def primary_key_prefix(table_name): prefix = primary_key_prefix(target_table) EXPECT_STDOUT_CONTAINS(f"{qualified_table_name()}: Records: 4079 Deleted: 0 Skipped: 4079 Warnings: 4079") -EXPECT_STDOUT_CONTAINS(f"WARNING: world_x_cities.csv error 1062: Duplicate entry '1' for key '{prefix}PRIMARY'") -EXPECT_STDOUT_CONTAINS(f"WARNING: world_x_cities.csv error 1062: Duplicate entry '2' for key '{prefix}PRIMARY'") -EXPECT_STDOUT_CONTAINS(f"WARNING: world_x_cities.csv error 1062: Duplicate entry '3' for key '{prefix}PRIMARY'") -EXPECT_STDOUT_CONTAINS(f"WARNING: world_x_cities.csv error 1062: Duplicate entry '4' for key '{prefix}PRIMARY'") -EXPECT_STDOUT_CONTAINS(f"WARNING: world_x_cities.csv error 1062: Duplicate entry '5' for key '{prefix}PRIMARY'") +EXPECT_STDOUT_CONTAINS(f"WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities.csv', error 1062: Duplicate entry '1' for key '{prefix}PRIMARY'") +EXPECT_STDOUT_CONTAINS(f"WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities.csv', error 1062: Duplicate entry '2' for key '{prefix}PRIMARY'") +EXPECT_STDOUT_CONTAINS(f"WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities.csv', error 1062: Duplicate entry '3' for key '{prefix}PRIMARY'") +EXPECT_STDOUT_CONTAINS(f"WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities.csv', error 1062: Duplicate entry '4' for key '{prefix}PRIMARY'") +EXPECT_STDOUT_CONTAINS(f"WARNING: LOAD DATA error for `wl12193`.`cities` in file 'world_x_cities.csv', error 1062: Duplicate entry '5' for key '{prefix}PRIMARY'") EXPECT_STDOUT_CONTAINS(f"File '{filename_for_output(world_x_cities_csv)}' (250.53 KB) was imported in ") EXPECT_STDOUT_CONTAINS(f"Total rows affected in {qualified_table_name()}: Records: 4079 Deleted: 0 Skipped: 4079 Warnings: 4079") EXPECT_EQ(target_table_checksum, md5_table(session, target_schema, target_table)) @@ -181,11 +181,11 @@ def primary_key_prefix(table_name): prefix = primary_key_prefix("document_store") EXPECT_STDOUT_CONTAINS(f"{target_schema}.document_store: Records: 25359 Deleted: 0 Skipped: 25359 Warnings: 25359") -EXPECT_STDOUT_CONTAINS(f"WARNING: primer-dataset-id.json error 1062: Duplicate entry '000000000001' for key '{prefix}PRIMARY'") -EXPECT_STDOUT_CONTAINS(f"WARNING: primer-dataset-id.json error 1062: Duplicate entry '000000000002' for key '{prefix}PRIMARY'") -EXPECT_STDOUT_CONTAINS(f"WARNING: primer-dataset-id.json error 1062: Duplicate entry '000000000003' for key '{prefix}PRIMARY'") -EXPECT_STDOUT_CONTAINS(f"WARNING: primer-dataset-id.json error 1062: Duplicate entry '000000000004' for key '{prefix}PRIMARY'") -EXPECT_STDOUT_CONTAINS(f"WARNING: primer-dataset-id.json error 1062: Duplicate entry '000000000005' for key '{prefix}PRIMARY'") +EXPECT_STDOUT_CONTAINS(f"WARNING: LOAD DATA error for `wl12193`.`document_store` in file 'primer-dataset-id.json', error 1062: Duplicate entry '000000000001' for key '{prefix}PRIMARY'") +EXPECT_STDOUT_CONTAINS(f"WARNING: LOAD DATA error for `wl12193`.`document_store` in file 'primer-dataset-id.json', error 1062: Duplicate entry '000000000002' for key '{prefix}PRIMARY'") +EXPECT_STDOUT_CONTAINS(f"WARNING: LOAD DATA error for `wl12193`.`document_store` in file 'primer-dataset-id.json', error 1062: Duplicate entry '000000000003' for key '{prefix}PRIMARY'") +EXPECT_STDOUT_CONTAINS(f"WARNING: LOAD DATA error for `wl12193`.`document_store` in file 'primer-dataset-id.json', error 1062: Duplicate entry '000000000004' for key '{prefix}PRIMARY'") +EXPECT_STDOUT_CONTAINS(f"WARNING: LOAD DATA error for `wl12193`.`document_store` in file 'primer-dataset-id.json', error 1062: Duplicate entry '000000000005' for key '{prefix}PRIMARY'") EXPECT_STDOUT_CONTAINS(f"File '{filename_for_output(primer_dataset_id)}' (11.29 MB) was imported in ") EXPECT_STDOUT_CONTAINS(f"Total rows affected in {target_schema}.document_store: Records: 25359 Deleted: 0 Skipped: 25359 Warnings: 25359") From 1120f19b8e0f475685ef5fdbd58273c49dacfcd8 Mon Sep 17 00:00:00 2001 From: Swati Gupta Date: Tue, 17 Jun 2025 11:16:37 +0200 Subject: [PATCH 2/2] Updated the LICENSE files --- LICENSE | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/LICENSE b/LICENSE index 3e22b9a4c3..0b88ef677c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ Licensing Information User Manual -MySQL Shell, part of MySQL 8.0.42 Community +MySQL Shell, part of MySQL 8.0.43 Community __________________________________________________________________ Introduction @@ -8,18 +8,18 @@ Introduction This License Information User Manual contains Oracle's product license and other licensing information, including licensing information for third-party software which may be included in this distribution of - MySQL Shell, part of MySQL 8.0.42 Community. + MySQL Shell, part of MySQL 8.0.43 Community. - Last updated: March 2025 + Last updated: June 2025 Licensing Information - This release of MySQL Shell, part of MySQL 8.0.42 Community, is brought + This release of MySQL Shell, part of MySQL 8.0.43 Community, is brought to you by the MySQL team at Oracle. This software is released under version 2 of the GNU General Public License (GPLv2), as set forth below, with the following additional permissions: - This distribution of MySQL Shell, part of MySQL 8.0.42 Community, is + This distribution of MySQL Shell, part of MySQL 8.0.43 Community, is designed to work with certain software (including but not limited to OpenSSL) that is licensed under separate terms, as designated in a particular file or component or in the license documentation. Without