From 6c682ff7180a3a4423d9917828a122efd224f455 Mon Sep 17 00:00:00 2001 From: Oleh Posyniak Date: Mon, 21 Oct 2019 12:41:11 -0500 Subject: [PATCH 1/8] MAGECLOUD-4458: De-compose All Patches from ECE-Tools --- .gitignore | 6 + .travis.yml | 18 + autoload.php | 19 + bin/.gitignore | 1 + bin/ece-patches | 10 + bootstrap.php | 10 + composer.json | 45 + config/services.xml | 17 + patches.json | 230 ++++ ..._import_during_ece_tools_dump__2.2.2.patch | 15 + ..._fix_session_manager_locking__2.1.10.patch | 24 + ...__fix_session_manager_locking__2.2.0.patch | 24 + ...igure_scd_on_demand_for_cloud__2.1.4.patch | 149 +++ ...igure_scd_on_demand_for_cloud__2.2.0.patch | 204 +++ ...601__trim_static_content_path__2.1.4.patch | 11 + ...overhaul_cron_implementation__2.1.13.patch | 907 +++++++++++++ ...overhaul_cron_implementation__2.1.14.patch | 902 +++++++++++++ ..._overhaul_cron_implementation__2.1.4.patch | 924 +++++++++++++ ..._overhaul_cron_implementation__2.1.5.patch | 924 +++++++++++++ ..._overhaul_cron_implementation__2.2.0.patch | 616 +++++++++ ..._overhaul_cron_implementation__2.2.2.patch | 595 +++++++++ ..._overhaul_cron_implementation__2.2.4.patch | 601 +++++++++ ...respect_minification_override__2.1.4.patch | 23 + ...respect_minification_override__2.2.0.patch | 20 + ...xml_and_robotstxt_generation__2.1.11.patch | 161 +++ ...pxml_and_robotstxt_generation__2.1.4.patch | 165 +++ ...event_deadlock_during_db_dump__2.2.0.patch | 13 + ...le_editing_when_scd_on_demand__2.2.0.patch | 345 +++++ ...rsion_error_during_deployment__2.2.0.patch | 16 + ...ating_factories_in_extensions__2.2.0.patch | 159 +++ ...7__resolve_issues_with_cron_schedule.patch | 28 + ..._run_cron_when_it_is_disabled__2.1.4.patch | 62 + ...mer_runners_on_cloud_clusters__2.2.0.patch | 39 + ...mer_runners_on_cloud_clusters__2.2.4.patch | 13 + ...check_for_console_application__2.2.0.patch | 61 + ...check_for_console_application__2.2.6.patch | 53 + ...OUD-2521__zendframework1_use_TLS_1.2.patch | 58 + ...lation_without_admin_creation__2.1.4.patch | 152 +++ ...lation_without_admin_creation__2.2.2.patch | 198 +++ ...ix_timezone_parsing_for_cron__2.1.13.patch | 122 ++ ...fix_timezone_parsing_for_cron__2.1.4.patch | 170 +++ ...fix_timezone_parsing_for_cron__2.1.5.patch | 170 +++ ...793__fix_monolog_slack_handler_2.1.x.patch | 164 +++ ...olated_connections_mechanism__2.1.13.patch | 76 ++ ...solated_connections_mechanism__2.1.4.patch | 79 ++ ...solated_connections_mechanism__2.1.5.patch | 79 ++ ...solated_connections_mechanism__2.2.0.patch | 75 ++ ...D-2822__configure_max_execution_time.patch | 107 ++ ...__configure_max_execution_time_2.3.1.patch | 89 ++ ...850_fix_amazon_payment_module__2.2.6.patch | 177 +++ ...ix_redis_slave_configuration__2.1.16.patch | 40 + ...fix_redis_slave_configuration__2.2.3.patch | 40 + ...fix_redis_slave_configuration__2.3.0.patch | 40 + ...add_zookeeper_and_flock_locks__2.2.5.patch | 1068 +++++++++++++++ ...add_zookeeper_and_flock_locks__2.3.0.patch | 1055 +++++++++++++++ ...ECLOUD-3611__multi_thread_scd__2.2.0.patch | 63 + ...ECLOUD-3611__multi_thread_scd__2.2.4.patch | 245 ++++ ...ECLOUD-3611__multi_thread_scd__2.3.0.patch | 63 + ...ECLOUD-3611__multi_thread_scd__2.3.2.patch | 71 + ...or_code_fix_for_setup_upgrade__2.2.0.patch | 32 + ...or_code_fix_for_setup_upgrade__2.2.1.patch | 33 + ...or_code_fix_for_setup_upgrade__2.3.0.patch | 15 + ...mer_runners_on_cloud_clusters__2.2.5.patch | 573 ++++++++ ...mer_runners_on_cloud_clusters__2.2.6.patch | 572 ++++++++ ...mer_runners_on_cloud_clusters__2.2.7.patch | 548 ++++++++ ...mer_runners_on_cloud_clusters__2.2.8.patch | 496 +++++++ ...mer_runners_on_cloud_clusters__2.3.0.patch | 511 +++++++ ...mer_runners_on_cloud_clusters__2.3.1.patch | 447 +++++++ ...nsumers_if_the_queue_is_empty__2.2.0.patch | 170 +++ ...nsumers_if_the_queue_is_empty__2.3.2.patch | 181 +++ ...unnecessary_permission_checks__2.1.4.patch | 43 + ...unnecessary_permission_checks__2.2.0.patch | 43 + ...89__load_appropriate_js_files__2.1.4.patch | 26 + ..._avoid_nonexistent_setup_area__2.1.4.patch | 15 + ..._fix_enterprise_payment_codes__2.1.8.patch | 93 ++ ...ont_skip_setup_scoped_plugins__2.1.4.patch | 20 + ...__move_vendor_path_autoloader__2.1.4.patch | 37 + ...atic_assets_without_rewrites__2.1.17.patch | 58 + ...tatic_assets_without_rewrites__2.1.4.patch | 58 + ...ent_excessive_js_optimization__2.1.4.patch | 521 ++++++++ ...x_scd_with_multiple_languages__2.1.4.patch | 13 + ...essary_write_permission_check__2.1.4.patch | 20 + ...7097__fix_credis_pipeline_bug__2.1.4.patch | 12 + ..._image_resizing_after_upgrade__2.1.6.patch | 187 +++ ...ort_credis_forking_during_scd__2.1.4.patch | 169 +++ ...ort_credis_forking_during_scd__2.2.0.patch | 169 +++ ...2__reload_js_translation_data__2.2.0.patch | 11 + ...-84444__fix_mview_on_staging__2.1.10.patch | 1190 +++++++++++++++++ ...O-84444__fix_mview_on_staging__2.1.4.patch | 1172 ++++++++++++++++ ...O-84444__fix_mview_on_staging__2.1.5.patch | 1190 +++++++++++++++++ ...ix_complex_folder_js_bundling__2.2.0.patch | 13 + ...ix_complex_folder_js_bundling__2.1.4.patch | 13 + ...heck_of_directory_permissions__2.1.4.patch | 20 + ...8833__turn_off_google_chart_api__2.x.patch | 141 ++ patches/MC-5964__preauth_sql__2.1.4.patch | 12 + patches/MC-5964__preauth_sql__2.2.0.patch | 92 ++ patches/MC-5964__preauth_sql__2.3.0.patch | 123 ++ ..._asset_locking_race_condition__2.1.4.patch | 109 ++ ..._asset_locking_race_condition__2.2.0.patch | 80 ++ ...y_encode_characters_in_emails__2.1.4.patch | 12 + ...xer_fails_with_large_catalogs__1.0.3.patch | 88 ++ ...ault_stock_view_in_storefront__1.0.3.patch | 82 ++ ..._configurable-product-indexer__1.0.3.patch | 102 ++ ...exer__grouped-product-indexer__1.0.3.patch | 106 ++ ..._source_item_indexer__indexer__1.0.3.patch | 99 ++ ...ce_item_indexer__reservations__1.0.3.patch | 12 + ...x_oom_during_customer_import__2.1.11.patch | 104 ++ ...ix_oom_during_customer_import__2.1.4.patch | 110 ++ ...ix_oom_during_customer_import__2.2.0.patch | 110 ++ ...ix_oom_during_customer_import__2.2.4.patch | 102 ++ src/App/Container.php | 96 ++ src/App/GenericException.php | 26 + src/Application.php | 46 + src/Command/Apply.php | 72 + src/Command/Patch/Manager.php | 185 +++ src/Command/Patch/ManagerException.php | 17 + src/Filesystem/DirectoryList.php | 58 + src/Filesystem/FileList.php | 35 + src/Patch/Applier.php | 172 +++ src/Patch/ApplierException.php | 17 + src/Shell/ProcessFactory.php | 41 + src/Test/Unit/Command/ApplyTest.php | 64 + src/Test/Unit/Command/Patch/ManagerTest.php | 229 ++++ .../Patch/_files/m2-hotfixes/patch1.patch | 0 .../Patch/_files/m2-hotfixes/patch2.patch | 0 .../Patch/_files/m2-hotfixes/readme.md | 0 .../Unit/Filesystem/DirectoryListTest.php | 67 + src/Test/Unit/Filesystem/FileListTest.php | 51 + src/Test/Unit/Filesystem/_files/.gitignore | 0 src/Test/Unit/Patch/ApplierTest.php | 298 +++++ .../Sniffs/Directives/StrictTypesSniff.php | 120 ++ .../Whitespace/MultipleEmptyLinesSniff.php | 51 + tests/static/phpcs-ruleset.xml | 30 + tests/static/phpmd-ruleset.xml | 48 + tests/unit/.gitignore | 1 + tests/unit/phpunit.xml.dist | 27 + 136 files changed, 23717 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 autoload.php create mode 100644 bin/.gitignore create mode 100755 bin/ece-patches create mode 100644 bootstrap.php create mode 100644 composer.json create mode 100644 config/services.xml create mode 100644 patches.json create mode 100644 patches/MAGECLOUD-1567__fix_import_during_ece_tools_dump__2.2.2.patch create mode 100644 patches/MAGECLOUD-1582__fix_session_manager_locking__2.1.10.patch create mode 100644 patches/MAGECLOUD-1582__fix_session_manager_locking__2.2.0.patch create mode 100644 patches/MAGECLOUD-1601__configure_scd_on_demand_for_cloud__2.1.4.patch create mode 100644 patches/MAGECLOUD-1601__configure_scd_on_demand_for_cloud__2.2.0.patch create mode 100644 patches/MAGECLOUD-1601__trim_static_content_path__2.1.4.patch create mode 100644 patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.13.patch create mode 100644 patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.14.patch create mode 100644 patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.4.patch create mode 100644 patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.5.patch create mode 100644 patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.0.patch create mode 100644 patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.2.patch create mode 100644 patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.4.patch create mode 100644 patches/MAGECLOUD-1736__respect_minification_override__2.1.4.patch create mode 100644 patches/MAGECLOUD-1736__respect_minification_override__2.2.0.patch create mode 100644 patches/MAGECLOUD-1998__unify_sitemapxml_and_robotstxt_generation__2.1.11.patch create mode 100644 patches/MAGECLOUD-1998__unify_sitemapxml_and_robotstxt_generation__2.1.4.patch create mode 100644 patches/MAGECLOUD-2033__prevent_deadlock_during_db_dump__2.2.0.patch create mode 100644 patches/MAGECLOUD-2159__unlock_locale_editing_when_scd_on_demand__2.2.0.patch create mode 100644 patches/MAGECLOUD-2173__the_recursion_error_during_deployment__2.2.0.patch create mode 100644 patches/MAGECLOUD-2209__write_logs_for_failed_process_of_generating_factories_in_extensions__2.2.0.patch create mode 100644 patches/MAGECLOUD-2427__resolve_issues_with_cron_schedule.patch create mode 100644 patches/MAGECLOUD-2445__do_not_run_cron_when_it_is_disabled__2.1.4.patch create mode 100644 patches/MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.0.patch create mode 100644 patches/MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.4.patch create mode 100644 patches/MAGECLOUD-2509__remove_permission_check_for_console_application__2.2.0.patch create mode 100644 patches/MAGECLOUD-2509__remove_permission_check_for_console_application__2.2.6.patch create mode 100644 patches/MAGECLOUD-2521__zendframework1_use_TLS_1.2.patch create mode 100644 patches/MAGECLOUD-2573__installation_without_admin_creation__2.1.4.patch create mode 100644 patches/MAGECLOUD-2573__installation_without_admin_creation__2.2.2.patch create mode 100644 patches/MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.13.patch create mode 100644 patches/MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.4.patch create mode 100644 patches/MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.5.patch create mode 100644 patches/MAGECLOUD-2793__fix_monolog_slack_handler_2.1.x.patch create mode 100644 patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.13.patch create mode 100644 patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.4.patch create mode 100644 patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.5.patch create mode 100644 patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.2.0.patch create mode 100644 patches/MAGECLOUD-2822__configure_max_execution_time.patch create mode 100644 patches/MAGECLOUD-2822__configure_max_execution_time_2.3.1.patch create mode 100644 patches/MAGECLOUD-2850_fix_amazon_payment_module__2.2.6.patch create mode 100644 patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.1.16.patch create mode 100644 patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.2.3.patch create mode 100644 patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.3.0.patch create mode 100644 patches/MAGECLOUD-3054__add_zookeeper_and_flock_locks__2.2.5.patch create mode 100644 patches/MAGECLOUD-3054__add_zookeeper_and_flock_locks__2.3.0.patch create mode 100644 patches/MAGECLOUD-3611__multi_thread_scd__2.2.0.patch create mode 100644 patches/MAGECLOUD-3611__multi_thread_scd__2.2.4.patch create mode 100644 patches/MAGECLOUD-3611__multi_thread_scd__2.3.0.patch create mode 100644 patches/MAGECLOUD-3611__multi_thread_scd__2.3.2.patch create mode 100644 patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.2.0.patch create mode 100644 patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.2.1.patch create mode 100644 patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.3.0.patch create mode 100644 patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.5.patch create mode 100644 patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.6.patch create mode 100644 patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.7.patch create mode 100644 patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.8.patch create mode 100644 patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.0.patch create mode 100644 patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.1.patch create mode 100644 patches/MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.2.0.patch create mode 100644 patches/MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.3.2.patch create mode 100644 patches/MAGECLOUD-414__remove_unnecessary_permission_checks__2.1.4.patch create mode 100644 patches/MAGECLOUD-414__remove_unnecessary_permission_checks__2.2.0.patch create mode 100644 patches/MAGECLOUD-589__load_appropriate_js_files__2.1.4.patch create mode 100644 patches/MAGETWO-45357__avoid_nonexistent_setup_area__2.1.4.patch create mode 100644 patches/MAGETWO-53941__fix_enterprise_payment_codes__2.1.8.patch create mode 100644 patches/MAGETWO-56675__dont_skip_setup_scoped_plugins__2.1.4.patch create mode 100644 patches/MAGETWO-57413__move_vendor_path_autoloader__2.1.4.patch create mode 100644 patches/MAGETWO-57414__load_static_assets_without_rewrites__2.1.17.patch create mode 100644 patches/MAGETWO-57414__load_static_assets_without_rewrites__2.1.4.patch create mode 100644 patches/MAGETWO-62660__prevent_excessive_js_optimization__2.1.4.patch create mode 100644 patches/MAGETWO-63020__fix_scd_with_multiple_languages__2.1.4.patch create mode 100644 patches/MAGETWO-63032__skip_unnecessary_write_permission_check__2.1.4.patch create mode 100644 patches/MAGETWO-67097__fix_credis_pipeline_bug__2.1.4.patch create mode 100644 patches/MAGETWO-67805__fix_image_resizing_after_upgrade__2.1.6.patch create mode 100644 patches/MAGETWO-69847__support_credis_forking_during_scd__2.1.4.patch create mode 100644 patches/MAGETWO-69847__support_credis_forking_during_scd__2.2.0.patch create mode 100644 patches/MAGETWO-82752__reload_js_translation_data__2.2.0.patch create mode 100644 patches/MAGETWO-84444__fix_mview_on_staging__2.1.10.patch create mode 100644 patches/MAGETWO-84444__fix_mview_on_staging__2.1.4.patch create mode 100644 patches/MAGETWO-84444__fix_mview_on_staging__2.1.5.patch create mode 100644 patches/MAGETWO-84507__fix_complex_folder_js_bundling__2.2.0.patch create mode 100644 patches/MAGETWO-88336__fix_complex_folder_js_bundling__2.1.4.patch create mode 100644 patches/MAGETWO-93265__fix_depth_of_recursive_check_of_directory_permissions__2.1.4.patch create mode 100644 patches/MAGETWO-98833__turn_off_google_chart_api__2.x.patch create mode 100644 patches/MC-5964__preauth_sql__2.1.4.patch create mode 100644 patches/MC-5964__preauth_sql__2.2.0.patch create mode 100644 patches/MC-5964__preauth_sql__2.3.0.patch create mode 100644 patches/MDVA-2470__fix_asset_locking_race_condition__2.1.4.patch create mode 100644 patches/MDVA-2470__fix_asset_locking_race_condition__2.2.0.patch create mode 100644 patches/MDVA-8695__properly_encode_characters_in_emails__2.1.4.patch create mode 100644 patches/MSI-2210__price_indexer_fails_with_large_catalogs__1.0.3.patch create mode 100644 patches/MSI-GH-2350__avoid_quering_inventory_default_stock_view_in_storefront__1.0.3.patch create mode 100644 patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__configurable-product-indexer__1.0.3.patch create mode 100644 patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__grouped-product-indexer__1.0.3.patch create mode 100644 patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__indexer__1.0.3.patch create mode 100644 patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__reservations__1.0.3.patch create mode 100644 patches/SET-36__fix_oom_during_customer_import__2.1.11.patch create mode 100644 patches/SET-36__fix_oom_during_customer_import__2.1.4.patch create mode 100644 patches/SET-36__fix_oom_during_customer_import__2.2.0.patch create mode 100644 patches/SET-36__fix_oom_during_customer_import__2.2.4.patch create mode 100644 src/App/Container.php create mode 100644 src/App/GenericException.php create mode 100644 src/Application.php create mode 100644 src/Command/Apply.php create mode 100644 src/Command/Patch/Manager.php create mode 100644 src/Command/Patch/ManagerException.php create mode 100644 src/Filesystem/DirectoryList.php create mode 100644 src/Filesystem/FileList.php create mode 100644 src/Patch/Applier.php create mode 100644 src/Patch/ApplierException.php create mode 100644 src/Shell/ProcessFactory.php create mode 100644 src/Test/Unit/Command/ApplyTest.php create mode 100644 src/Test/Unit/Command/Patch/ManagerTest.php create mode 100644 src/Test/Unit/Command/Patch/_files/m2-hotfixes/patch1.patch create mode 100644 src/Test/Unit/Command/Patch/_files/m2-hotfixes/patch2.patch create mode 100644 src/Test/Unit/Command/Patch/_files/m2-hotfixes/readme.md create mode 100644 src/Test/Unit/Filesystem/DirectoryListTest.php create mode 100644 src/Test/Unit/Filesystem/FileListTest.php create mode 100644 src/Test/Unit/Filesystem/_files/.gitignore create mode 100644 src/Test/Unit/Patch/ApplierTest.php create mode 100644 tests/static/Sniffs/Directives/StrictTypesSniff.php create mode 100644 tests/static/Sniffs/Whitespace/MultipleEmptyLinesSniff.php create mode 100644 tests/static/phpcs-ruleset.xml create mode 100644 tests/static/phpmd-ruleset.xml create mode 100644 tests/unit/.gitignore create mode 100644 tests/unit/phpunit.xml.dist diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a3b63ba --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.DS_Store +/.idea +/vendor +/composer.phar +/composer.lock +/auth.json diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..1104c7c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +dist: xenial + +git: + depth: false + +language: php +php: + - '7.0' + - '7.1' + - '7.2' + - '7.3' + +install: composer update + +script: + - ./vendor/bin/phpcs ./src --standard=./tests/static/phpcs-ruleset.xml -p -n + - ./vendor/bin/phpmd ./src xml ./tests/static/phpmd-ruleset.xml + - ./vendor/bin/phpunit --configuration ./tests/unit diff --git a/autoload.php b/autoload.php new file mode 100644 index 0000000..8cff697 --- /dev/null +++ b/autoload.php @@ -0,0 +1,19 @@ +run(); diff --git a/bootstrap.php b/bootstrap.php new file mode 100644 index 0000000..f608152 --- /dev/null +++ b/bootstrap.php @@ -0,0 +1,10 @@ + + + + + + + + + + + + + + + + + diff --git a/patches.json b/patches.json new file mode 100644 index 0000000..0a5ec42 --- /dev/null +++ b/patches.json @@ -0,0 +1,230 @@ +{ + "magento/magento2-base": { + "Fix asset locker race condition when using Redis": { + "2.1.4 - 2.1.14": "MDVA-2470__fix_asset_locking_race_condition__2.1.4.patch", + "2.2.0 - 2.2.5": "MDVA-2470__fix_asset_locking_race_condition__2.2.0.patch" + }, + "Prevent redundant permissions check during build": { + "2.1.4 - 2.1.14": "MAGECLOUD-414__remove_unnecessary_permission_checks__2.1.4.patch", + "2.2.0 - 2.2.5": "MAGECLOUD-414__remove_unnecessary_permission_checks__2.2.0.patch" + }, + "Fix Redis issues with session manager locking": { + "2.1.10 - 2.1.13" : "MAGECLOUD-1582__fix_session_manager_locking__2.1.10.patch", + "2.2.0 - 2.2.1": "MAGECLOUD-1582__fix_session_manager_locking__2.2.0.patch" + }, + "Workaround app/etc not being available before the deploy phase": { + "~2.1.4": "MAGETWO-57413__move_vendor_path_autoloader__2.1.4.patch" + }, + "Allow static assets to be loaded without URL rewrites": { + "2.1.4 - 2.1.16": "MAGETWO-57414__load_static_assets_without_rewrites__2.1.4.patch", + "~2.1.17": "MAGETWO-57414__load_static_assets_without_rewrites__2.1.17.patch" + }, + "Don't attempt to use non-existent setup areas": { + "~2.1.4": "MAGETWO-45357__avoid_nonexistent_setup_area__2.1.4.patch" + }, + "Skip checking var/generation for write permissions when it doesn't affect the build process": { + "2.1.4 - 2.1.14": "MAGETWO-63032__skip_unnecessary_write_permission_check__2.1.4.patch" + }, + "Fix loading multiple plugins in the setup scope": { + "~2.1.4": "MAGETWO-56675__dont_skip_setup_scoped_plugins__2.1.4.patch" + }, + "Support SCD forking in the credis connector": { + "~2.1.4": "MAGETWO-69847__support_credis_forking_during_scd__2.1.4.patch", + "2.2.0 - 2.2.5" : "MAGETWO-69847__support_credis_forking_during_scd__2.2.0.patch" + }, + "Allow multiple languages to be specified for the SCD command": { + "2.1.4 - 2.1.7": "MAGETWO-63020__fix_scd_with_multiple_languages__2.1.4.patch" + }, + "Continue to load javascript assets in the admin panel when using particular build parameters": { + "2.1.4 - 2.1.7": "MAGECLOUD-589__load_appropriate_js_files__2.1.4.patch" + }, + "Handle special characters in email headers": { + "~2.1.4": "MDVA-8695__properly_encode_characters_in_emails__2.1.4.patch", + "2.2.0": "MDVA-8695__properly_encode_characters_in_emails__2.1.4.patch" + }, + "Enable SCD on demand in production": { + ">=2.1.4": "MAGECLOUD-1601__trim_static_content_path__2.1.4.patch", + "~2.1.4": "MAGECLOUD-1601__configure_scd_on_demand_for_cloud__2.1.4.patch", + "2.2.0 - 2.2.3": "MAGECLOUD-1601__configure_scd_on_demand_for_cloud__2.2.0.patch" + }, + "Respect user-specified minification settings": { + "~2.1.4": "MAGECLOUD-1736__respect_minification_override__2.1.4.patch", + "~2.2.0": "MAGECLOUD-1736__respect_minification_override__2.2.0.patch" + }, + "Process the application cron queue more reliably": { + "2.1.4": "MAGECLOUD-1607__overhaul_cron_implementation__2.1.4.patch", + "2.1.5 - 2.1.12": "MAGECLOUD-1607__overhaul_cron_implementation__2.1.5.patch", + "2.1.13": "MAGECLOUD-1607__overhaul_cron_implementation__2.1.13.patch", + "~2.1.14": "MAGECLOUD-1607__overhaul_cron_implementation__2.1.14.patch", + "2.2.0 - 2.2.1": "MAGECLOUD-1607__overhaul_cron_implementation__2.2.0.patch", + "2.2.2 - 2.2.3": "MAGECLOUD-1607__overhaul_cron_implementation__2.2.2.patch", + "2.2.4": "MAGECLOUD-1607__overhaul_cron_implementation__2.2.4.patch" + }, + "Add Zookeeper and flock locks": { + "2.2.5 - 2.2.8": "MAGECLOUD-3054__add_zookeeper_and_flock_locks__2.2.5.patch", + "2.3.0 - 2.3.1": "MAGECLOUD-3054__add_zookeeper_and_flock_locks__2.3.0.patch" + }, + "Reduce memory usage when importing customers and addresses": { + "2.1.4 - 2.1.10": "SET-36__fix_oom_during_customer_import__2.1.4.patch", + "2.1.11 - 2.1.12": "SET-36__fix_oom_during_customer_import__2.1.11.patch", + "2.2.0 - 2.2.3": "SET-36__fix_oom_during_customer_import__2.2.0.patch", + "2.2.4": "SET-36__fix_oom_during_customer_import__2.2.4.patch" + }, + "Add PayPal and Braintree TPV codes on checkout": { + "~2.1.8": "MAGETWO-53941__fix_enterprise_payment_codes__2.1.8.patch" + }, + "Fix Mview on staging environments": { + "2.1.4": "MAGETWO-84444__fix_mview_on_staging__2.1.4.patch", + "2.1.5 - 2.1.9": "MAGETWO-84444__fix_mview_on_staging__2.1.5.patch", + "2.1.10": "MAGETWO-84444__fix_mview_on_staging__2.1.10.patch" + }, + "Resize images properly after upgrading to 2.1.6": { + "2.1.6": "MAGETWO-67805__fix_image_resizing_after_upgrade__2.1.6.patch" + }, + "Bundle javascript files even when other files are present": { + "2.1.4 - 2.1.12": "MAGETWO-88336__fix_complex_folder_js_bundling__2.1.4.patch", + "2.2.0 - 2.2.3": "MAGETWO-84507__fix_complex_folder_js_bundling__2.2.0.patch" + }, + "Allow ece-tools dumps to complete by fixing app:config:import": { + "2.2.2": "MAGECLOUD-1567__fix_import_during_ece_tools_dump__2.2.2.patch" + }, + "Unify robots.txt and sitemap.xml generation": { + "2.1.4 - 2.1.10": "MAGECLOUD-1998__unify_sitemapxml_and_robotstxt_generation__2.1.4.patch", + "~2.1.11": "MAGECLOUD-1998__unify_sitemapxml_and_robotstxt_generation__2.1.11.patch" + }, + "Fix javascript localization issues": { + "~2.1.4": "MAGETWO-62660__prevent_excessive_js_optimization__2.1.4.patch", + "2.2.0 - 2.2.1": "MAGETWO-82752__reload_js_translation_data__2.2.0.patch" + }, + "Unlock locale editing when SCD on demand is enabled": { + "2.2.0 - 2.2.5": "MAGECLOUD-2159__unlock_locale_editing_when_scd_on_demand__2.2.0.patch" + }, + "Allow DB dumps done with the support module to complete": { + "2.2.0 - 2.2.5": "MAGECLOUD-2033__prevent_deadlock_during_db_dump__2.2.0.patch" + }, + "Write Logs for Failed Process of Generating Factories in Extensions": { + "2.2.0 - 2.2.5": "MAGECLOUD-2209__write_logs_for_failed_process_of_generating_factories_in_extensions__2.2.0.patch" + }, + "Fix Problems with Consumer Runners on Cloud Clusters": { + "2.2.0 - 2.2.3": "MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.0.patch", + "2.2.4": "MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.4.patch", + "2.2.5": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.5.patch", + "2.2.6": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.6.patch", + "2.2.7": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.7.patch", + "2.2.8 - 2.2.9": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.8.patch", + "2.3.0": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.0.patch", + "2.3.1 - 2.3.2": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.1.patch" + }, + "Resolve Issues with Cron Schedule": { + "2.1.10 - 2.1.14 || 2.2.2 - 2.2.5": "MAGECLOUD-2427__resolve_issues_with_cron_schedule.patch" + }, + "Fix timezone parsing for Cron": { + "2.1.4": "MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.4.patch", + "2.1.5 - 2.1.10": "MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.5.patch", + "2.1.13 - 2.1.14": "MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.13.patch" + }, + "Change the depth of a recursive check of directory write permissions": { + "2.1.4 - 2.1.14": "MAGETWO-93265__fix_depth_of_recursive_check_of_directory_permissions__2.1.4.patch" + }, + "Google chart API used by Magento dashboard scheduled to be turned off": { + "2.1.4 - 2.1.17": "MAGETWO-98833__turn_off_google_chart_api__2.x.patch", + "2.2.0 - 2.2.8": "MAGETWO-98833__turn_off_google_chart_api__2.x.patch", + "2.3.0 - 2.3.1": "MAGETWO-98833__turn_off_google_chart_api__2.x.patch" + }, + "Do not run cron when it is disabled": { + "2.1.4 - 2.2.5": "MAGECLOUD-2445__do_not_run_cron_when_it_is_disabled__2.1.4.patch" + }, + "Zendframework1 should use TLS 1.2": { + ">=2.1.4 <2.3": "MAGECLOUD-2521__zendframework1_use_TLS_1.2.patch" + }, + "The recursion detected error during deployment": { + "2.2.0 - 2.2.6": "MAGECLOUD-2173__the_recursion_error_during_deployment__2.2.0.patch" + }, + "Remove the permission check for the console application": { + "2.2.0 - 2.2.5": "MAGECLOUD-2509__remove_permission_check_for_console_application__2.2.0.patch", + "2.2.6": "MAGECLOUD-2509__remove_permission_check_for_console_application__2.2.6.patch" + }, + "Fix for DI compilation with Amazon_Payment module": { + "2.2.6": "MAGECLOUD-2850_fix_amazon_payment_module__2.2.6.patch" + }, + "Add the possibility to install Magento without admin creation" : { + "2.1.4 - 2.2.1": "MAGECLOUD-2573__installation_without_admin_creation__2.1.4.patch", + "2.2.2 - 2.2.7": "MAGECLOUD-2573__installation_without_admin_creation__2.2.2.patch" + }, + "Add the possibility to configure max execution time during static content deployment": { + "2.2.0 - 2.2.8 || 2.3.0": "MAGECLOUD-2822__configure_max_execution_time.patch", + "2.3.1": "MAGECLOUD-2822__configure_max_execution_time_2.3.1.patch" + }, + "Suppress PDO warnings to work around PHP bugs #63812, #74401": { + "2.1.4": "MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.4.patch", + "2.1.5 - 2.1.12": "MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.5.patch", + "2.1.13 - 2.1.17": "MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.13.patch", + "2.2.0 - 2.2.8 || 2.3.0 - 2.3.1": "MAGECLOUD-2820__implement_isolated_connections_mechanism__2.2.0.patch" + }, + "Pre-auth SQL": { + "2.1.4 - 2.1.17": "MC-5964__preauth_sql__2.1.4.patch", + "2.2.0 - 2.2.7": "MC-5964__preauth_sql__2.2.0.patch", + "2.3.0": "MC-5964__preauth_sql__2.3.0.patch" + }, + "Multi-thread SCD": { + "2.2.0 - 2.2.3": "MAGECLOUD-3611__multi_thread_scd__2.2.0.patch", + "2.2.4 - 2.2.9": "MAGECLOUD-3611__multi_thread_scd__2.2.4.patch", + "2.3.0 - 2.3.1": "MAGECLOUD-3611__multi_thread_scd__2.3.0.patch", + "2.3.2": "MAGECLOUD-3611__multi_thread_scd__2.3.2.patch" + }, + "setup:upgrade returns error code if app:config:import failed": { + "2.2.0": "MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.2.0.patch", + "2.2.1 - 2.2.9": "MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.2.1.patch", + "2.3.0 - 2.3.2": "MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.3.0.patch" + }, + "Re-work consumers to terminate as soon as there is nothing left to process": { + "2.2.0 - 2.3.1": "MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.2.0.patch", + "2.3.2 - 2.3.3": "MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.3.2.patch" + } + }, + "monolog/monolog": { + "Fix monolog Slack Handler bug for magento 2.1.x": { + "1.16.0": "MAGECLOUD-2793__fix_monolog_slack_handler_2.1.x.patch" + } + }, + "colinmollenhour/cache-backend-redis": { + "The ability to read from the master Redis instance if the slave Redis is unavailable:": { + "1.10.2": "MAGECLOUD-2899__fix_redis_slave_configuration__2.1.16.patch", + "1.10.4": "MAGECLOUD-2899__fix_redis_slave_configuration__2.2.3.patch", + "1.10.5": "MAGECLOUD-2899__fix_redis_slave_configuration__2.3.0.patch" + } + }, + "colinmollenhour/credis": { + "Fix credis pipeline issue": { + "1.6": "MAGETWO-67097__fix_credis_pipeline_bug__2.1.4.patch" + } + }, + "magento/module-inventory-catalog": { + "Price indexer fails with large catalogs": { + ">=1.0.3 <1.0.6": "MSI-2210__price_indexer_fails_with_large_catalogs__1.0.3.patch" + } + }, + "magento/module-inventory-indexer": { + "Avoid quering inventory default stock view in storefront": { + ">=1.0.3 <1.0.6": "MSI-GH-2350__avoid_quering_inventory_default_stock_view_in_storefront__1.0.3.patch" + }, + "Avoid group concat from source item indexer": { + ">=1.0.3 <1.0.6": "MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__indexer__1.0.3.patch" + } + }, + "magento/module-inventory-reservations": { + "Avoid group concat from source item indexer": { + ">=1.0.3 <1.0.6": "MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__reservations__1.0.3.patch" + } + }, + "magento/module-inventory-configurable-product-indexer": { + "Avoid group concat from source item indexer": { + ">=1.0.3 <1.0.5": "MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__configurable-product-indexer__1.0.3.patch" + } + }, + "magento/module-inventory-grouped-product-indexer": { + "Avoid group concat from source item indexer": { + ">=1.0.3 <1.0.5": "MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__grouped-product-indexer__1.0.3.patch" + } + } +} diff --git a/patches/MAGECLOUD-1567__fix_import_during_ece_tools_dump__2.2.2.patch b/patches/MAGECLOUD-1567__fix_import_during_ece_tools_dump__2.2.2.patch new file mode 100644 index 0000000..ad1532e --- /dev/null +++ b/patches/MAGECLOUD-1567__fix_import_during_ece_tools_dump__2.2.2.patch @@ -0,0 +1,15 @@ +diff -Nuar a/vendor/magento/module-config/Model/Config/Importer.php b/vendor/magento/module-config/Model/Config/Importer.php +--- a/vendor/magento/module-config/Model/Config/Importer.php ++++ b/vendor/magento/module-config/Model/Config/Importer.php +@@ -129,8 +129,10 @@ class Importer implements ImporterInterface + + // Invoke saving of new values. + $this->saveProcessor->process($changedData); +- $this->flagManager->saveFlag(static::FLAG_CODE, $data); + }); ++ ++ $this->scope->setCurrentScope($currentScope); ++ $this->flagManager->saveFlag(static::FLAG_CODE, $data); + } catch (\Exception $e) { + throw new InvalidTransitionException(__('%1', $e->getMessage()), $e); + } finally { diff --git a/patches/MAGECLOUD-1582__fix_session_manager_locking__2.1.10.patch b/patches/MAGECLOUD-1582__fix_session_manager_locking__2.1.10.patch new file mode 100644 index 0000000..0fbdef8 --- /dev/null +++ b/patches/MAGECLOUD-1582__fix_session_manager_locking__2.1.10.patch @@ -0,0 +1,24 @@ +diff -Nuar a/vendor/magento/framework/Session/SessionManager.php b/vendor/magento/framework/Session/SessionManager.php +--- a/vendor/magento/framework/Session/SessionManager.php ++++ b/vendor/magento/framework/Session/SessionManager.php +@@ -470,18 +470,9 @@ class SessionManager implements SessionManagerInterface + if (headers_sent()) { + return $this; + } +- //@see http://php.net/manual/en/function.session-regenerate-id.php#53480 workaround ++ + if ($this->isSessionExists()) { +- $oldSessionId = session_id(); +- session_regenerate_id(); +- $newSessionId = session_id(); +- session_id($oldSessionId); +- session_destroy(); +- +- $oldSession = $_SESSION; +- session_id($newSessionId); +- session_start(); +- $_SESSION = $oldSession; ++ session_regenerate_id(true); + } else { + session_start(); + } diff --git a/patches/MAGECLOUD-1582__fix_session_manager_locking__2.2.0.patch b/patches/MAGECLOUD-1582__fix_session_manager_locking__2.2.0.patch new file mode 100644 index 0000000..33635d7 --- /dev/null +++ b/patches/MAGECLOUD-1582__fix_session_manager_locking__2.2.0.patch @@ -0,0 +1,24 @@ +diff -Nuar a/vendor/magento/framework/Session/SessionManager.php b/vendor/magento/framework/Session/SessionManager.php +index 2cea02f..272d3d9 100644 +--- a/vendor/magento/framework/Session/SessionManager.php ++++ b/vendor/magento/framework/Session/SessionManager.php +@@ -504,18 +504,8 @@ class SessionManager implements SessionManagerInterface + return $this; + } + +- //@see http://php.net/manual/en/function.session-regenerate-id.php#53480 workaround + if ($this->isSessionExists()) { +- $oldSessionId = session_id(); +- session_regenerate_id(); +- $newSessionId = session_id(); +- session_id($oldSessionId); +- session_destroy(); +- +- $oldSession = $_SESSION; +- session_id($newSessionId); +- session_start(); +- $_SESSION = $oldSession; ++ session_regenerate_id(true); + } else { + session_start(); + } diff --git a/patches/MAGECLOUD-1601__configure_scd_on_demand_for_cloud__2.1.4.patch b/patches/MAGECLOUD-1601__configure_scd_on_demand_for_cloud__2.1.4.patch new file mode 100644 index 0000000..fe8b750 --- /dev/null +++ b/patches/MAGECLOUD-1601__configure_scd_on_demand_for_cloud__2.1.4.patch @@ -0,0 +1,149 @@ +diff -Nuar a/vendor/magento/framework/Config/ConfigOptionsListConstants.php b/vendor/magento/framework/Config/ConfigOptionsListConstants.php +--- a/vendor/magento/framework/Config/ConfigOptionsListConstants.php ++++ b/vendor/magento/framework/Config/ConfigOptionsListConstants.php +@@ -11,6 +11,8 @@ namespace Magento\Framework\Config; + */ + class ConfigOptionsListConstants + { ++ const CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION = 'static_content_on_demand_in_production'; ++ + /**#@+ + * Path to the values in the deployment config + */ + +diff -Nuar a/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php b/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php +--- a/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php ++++ b/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php +@@ -10,6 +10,9 @@ use Magento\Framework\App\State; + use Magento\Framework\View\Asset\ConfigInterface; + use Magento\Framework\View\Design\ThemeInterface; + use Magento\Framework\View\Template\Html\MinifierInterface; ++use Magento\Framework\App\DeploymentConfig; ++use Magento\Framework\App\ObjectManager; ++use Magento\Framework\Config\ConfigOptionsListConstants; + + /** + * Provider of template view files +@@ -32,20 +35,28 @@ class TemplateFile extends File + protected $assetConfig; + + /** ++ * @var DeploymentConfig ++ */ ++ private $deploymentConfig; ++ ++ /** + * @param ResolverInterface $resolver + * @param MinifierInterface $templateMinifier + * @param State $appState + * @param ConfigInterface $assetConfig ++ * @param DeploymentConfig $deploymentConfig + */ + public function __construct( + ResolverInterface $resolver, + MinifierInterface $templateMinifier, + State $appState, +- ConfigInterface $assetConfig ++ ConfigInterface $assetConfig, ++ DeploymentConfig $deploymentConfig = null + ) { + $this->appState = $appState; + $this->templateMinifier = $templateMinifier; + $this->assetConfig = $assetConfig; ++ $this->deploymentConfig = $deploymentConfig ?: ObjectManager::getInstance()->get(DeploymentConfig::class); + parent::__construct($resolver); + } + +@@ -73,7 +84,7 @@ class TemplateFile extends File + if ($template && $this->assetConfig->isMinifyHtml()) { + switch ($this->appState->getMode()) { + case State::MODE_PRODUCTION: +- return $this->templateMinifier->getPathToMinified($template); ++ return $this->getMinifiedTemplateInProduction($template); + case State::MODE_DEFAULT: + return $this->templateMinifier->getMinified($template); + case State::MODE_DEVELOPER: +@@ -83,4 +94,24 @@ class TemplateFile extends File + } + return $template; + } ++ ++ /** ++ * Returns path to minified template file ++ * ++ * If SCD on demand in production is disabled - returns the path to minified template file. ++ * Otherwise returns the path to minified template file, ++ * or minify if file not exist and returns path. ++ * ++ * @param string $template ++ * @return string ++ */ ++ private function getMinifiedTemplateInProduction($template) ++ { ++ if ($this->deploymentConfig->getConfigData( ++ ConfigOptionsListConstants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION ++ )) { ++ return $this->templateMinifier->getMinified($template); ++ } ++ return $this->templateMinifier->getPathToMinified($template); ++ } + } + +diff -Nuar a/vendor/magento/framework/App/View/Deployment/Version.php b/vendor/magento/framework/App/View/Deployment/Version.php +--- a/vendor/magento/framework/App/View/Deployment/Version.php ++++ b/vendor/magento/framework/App/View/Deployment/Version.php +@@ -6,6 +6,10 @@ + + namespace Magento\Framework\App\View\Deployment; + ++use Magento\Framework\App\DeploymentConfig; ++use Magento\Framework\App\ObjectManager; ++use Magento\Framework\Config\ConfigOptionsListConstants; ++ + /** + * Deployment version of static files + */ +@@ -27,15 +31,23 @@ class Version + private $cachedValue; + + /** ++ * @var DeploymentConfig ++ */ ++ private $deploymentConfig; ++ ++ /** + * @param \Magento\Framework\App\State $appState + * @param Version\StorageInterface $versionStorage ++ * @param DeploymentConfig|null $deploymentConfig + */ + public function __construct( + \Magento\Framework\App\State $appState, +- \Magento\Framework\App\View\Deployment\Version\StorageInterface $versionStorage ++ \Magento\Framework\App\View\Deployment\Version\StorageInterface $versionStorage, ++ DeploymentConfig $deploymentConfig = null + ) { + $this->appState = $appState; + $this->versionStorage = $versionStorage; ++ $this->deploymentConfig = $deploymentConfig ?: ObjectManager::getInstance()->get(DeploymentConfig::class); + } + + /** +@@ -74,7 +86,17 @@ class Version + break; + + default: +- $result = $this->versionStorage->load(); ++ try { ++ $result = $this->versionStorage->load(); ++ } catch (\UnexpectedValueException $e) { ++ if (!$this->deploymentConfig->getConfigData( ++ ConfigOptionsListConstants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION ++ )) { ++ throw $e; ++ } ++ $result = (new \DateTime())->getTimestamp(); ++ $this->versionStorage->save($result); ++ } + } + return $result; + } diff --git a/patches/MAGECLOUD-1601__configure_scd_on_demand_for_cloud__2.2.0.patch b/patches/MAGECLOUD-1601__configure_scd_on_demand_for_cloud__2.2.0.patch new file mode 100644 index 0000000..f9d7a30 --- /dev/null +++ b/patches/MAGECLOUD-1601__configure_scd_on_demand_for_cloud__2.2.0.patch @@ -0,0 +1,204 @@ +diff -Nuar a/vendor/magento/framework/App/StaticResource.php b/vendor/magento/framework/App/StaticResource.php +--- a/vendor/magento/framework/App/StaticResource.php ++++ b/vendor/magento/framework/App/StaticResource.php +@@ -8,6 +8,7 @@ namespace Magento\Framework\App; + use Magento\Framework\App\Filesystem\DirectoryList; + use Magento\Framework\ObjectManager\ConfigLoaderInterface; + use Magento\Framework\Filesystem; ++use Magento\Framework\Config\ConfigOptionsListConstants; + use Psr\Log\LoggerInterface; + + /** +@@ -63,6 +64,11 @@ class StaticResource implements \Magento\Framework\AppInterface + private $filesystem; + + /** ++ * @var DeploymentConfig ++ */ ++ private $deploymentConfig; ++ ++ /** + * @var \Psr\Log\LoggerInterface + */ + private $logger; +@@ -76,6 +82,7 @@ class StaticResource implements \Magento\Framework\AppInterface + * @param \Magento\Framework\Module\ModuleList $moduleList + * @param \Magento\Framework\ObjectManagerInterface $objectManager + * @param ConfigLoaderInterface $configLoader ++ * @param DeploymentConfig|null $deploymentConfig + */ + public function __construct( + State $state, +@@ -85,7 +92,8 @@ class StaticResource implements \Magento\Framework\AppInterface + \Magento\Framework\View\Asset\Repository $assetRepo, + \Magento\Framework\Module\ModuleList $moduleList, + \Magento\Framework\ObjectManagerInterface $objectManager, +- ConfigLoaderInterface $configLoader ++ ConfigLoaderInterface $configLoader, ++ DeploymentConfig $deploymentConfig = null + ) { + $this->state = $state; + $this->response = $response; +@@ -95,6 +103,7 @@ class StaticResource implements \Magento\Framework\AppInterface + $this->moduleList = $moduleList; + $this->objectManager = $objectManager; + $this->configLoader = $configLoader; ++ $this->deploymentConfig = $deploymentConfig ?: ObjectManager::getInstance()->get(DeploymentConfig::class); + } + + /** +@@ -108,7 +117,11 @@ class StaticResource implements \Magento\Framework\AppInterface + // disabling profiling when retrieving static resource + \Magento\Framework\Profiler::reset(); + $appMode = $this->state->getMode(); +- if ($appMode == \Magento\Framework\App\State::MODE_PRODUCTION) { ++ if ($appMode == \Magento\Framework\App\State::MODE_PRODUCTION ++ && !$this->deploymentConfig->getConfigData( ++ ConfigOptionsListConstants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION ++ ) ++ ) { + $this->response->setHttpResponseCode(404); + } else { + $path = $this->request->get('resource'); + +diff -Nuar a/vendor/magento/framework/App/View/Deployment/Version.php b/vendor/magento/framework/App/View/Deployment/Version.php +--- a/vendor/magento/framework/App/View/Deployment/Version.php ++++ b/vendor/magento/framework/App/View/Deployment/Version.php +@@ -6,6 +6,9 @@ + + namespace Magento\Framework\App\View\Deployment; + ++use Magento\Framework\App\DeploymentConfig; ++use Magento\Framework\App\ObjectManager; ++use Magento\Framework\Config\ConfigOptionsListConstants; + use Psr\Log\LoggerInterface; + + /** +@@ -34,15 +37,23 @@ class Version + private $logger; + + /** ++ * @var DeploymentConfig ++ */ ++ private $deploymentConfig; ++ ++ /** + * @param \Magento\Framework\App\State $appState + * @param Version\StorageInterface $versionStorage ++ * @param DeploymentConfig|null $deploymentConfig + */ + public function __construct( + \Magento\Framework\App\State $appState, +- \Magento\Framework\App\View\Deployment\Version\StorageInterface $versionStorage ++ \Magento\Framework\App\View\Deployment\Version\StorageInterface $versionStorage, ++ DeploymentConfig $deploymentConfig = null + ) { + $this->appState = $appState; + $this->versionStorage = $versionStorage; ++ $this->deploymentConfig = $deploymentConfig ?: ObjectManager::getInstance()->get(DeploymentConfig::class); + } + + /** +@@ -68,7 +79,11 @@ class Version + { + $result = $this->versionStorage->load(); + if (!$result) { +- if ($appMode == \Magento\Framework\App\State::MODE_PRODUCTION) { ++ if ($appMode == \Magento\Framework\App\State::MODE_PRODUCTION ++ && !$this->deploymentConfig->getConfigData( ++ ConfigOptionsListConstants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION ++ ) ++ ) { + $this->getLogger()->critical('Can not load static content version.'); + throw new \UnexpectedValueException( + "Unable to retrieve deployment version of static files from the file system." + +diff -Nuar a/vendor/magento/framework/Config/ConfigOptionsListConstants.php b/vendor/magento/framework/Config/ConfigOptionsListConstants.php +--- a/vendor/magento/framework/Config/ConfigOptionsListConstants.php ++++ b/vendor/magento/framework/Config/ConfigOptionsListConstants.php +@@ -36,6 +36,7 @@ class ConfigOptionsListConstants + const CONFIG_PATH_DB_LOGGER_LOG_EVERYTHING = 'db_logger/log_everything'; + const CONFIG_PATH_DB_LOGGER_QUERY_TIME_THRESHOLD = 'db_logger/query_time_threshold'; + const CONFIG_PATH_DB_LOGGER_INCLUDE_STACKTRACE = 'db_logger/include_stacktrace'; ++ const CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION = 'static_content_on_demand_in_production'; + /**#@-*/ + + /**#@+ + +diff -Nuar a/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php b/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php +--- a/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php ++++ b/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php +@@ -10,6 +10,9 @@ use Magento\Framework\App\State; + use Magento\Framework\View\Asset\ConfigInterface; + use Magento\Framework\View\Design\ThemeInterface; + use Magento\Framework\View\Template\Html\MinifierInterface; ++use Magento\Framework\App\DeploymentConfig; ++use Magento\Framework\App\ObjectManager; ++use Magento\Framework\Config\ConfigOptionsListConstants; + + /** + * Provider of template view files +@@ -32,20 +35,28 @@ class TemplateFile extends File + protected $assetConfig; + + /** ++ * @var DeploymentConfig ++ */ ++ private $deploymentConfig; ++ ++ /** + * @param ResolverInterface $resolver + * @param MinifierInterface $templateMinifier + * @param State $appState + * @param ConfigInterface $assetConfig ++ * @param DeploymentConfig $deploymentConfig + */ + public function __construct( + ResolverInterface $resolver, + MinifierInterface $templateMinifier, + State $appState, +- ConfigInterface $assetConfig ++ ConfigInterface $assetConfig, ++ DeploymentConfig $deploymentConfig = null + ) { + $this->appState = $appState; + $this->templateMinifier = $templateMinifier; + $this->assetConfig = $assetConfig; ++ $this->deploymentConfig = $deploymentConfig ?: ObjectManager::getInstance()->get(DeploymentConfig::class); + parent::__construct($resolver); + } + +@@ -73,7 +84,7 @@ class TemplateFile extends File + if ($template && $this->assetConfig->isMinifyHtml()) { + switch ($this->appState->getMode()) { + case State::MODE_PRODUCTION: +- return $this->templateMinifier->getPathToMinified($template); ++ return $this->getMinifiedTemplateInProduction($template); + case State::MODE_DEFAULT: + return $this->templateMinifier->getMinified($template); + case State::MODE_DEVELOPER: +@@ -83,4 +94,24 @@ class TemplateFile extends File + } + return $template; + } ++ ++ /** ++ * Returns path to minified template file ++ * ++ * If SCD on demand in production is disabled - returns the path to minified template file. ++ * Otherwise returns the path to minified template file, ++ * or minify if file not exist and returns path. ++ * ++ * @param string $template ++ * @return string ++ */ ++ private function getMinifiedTemplateInProduction($template) ++ { ++ if ($this->deploymentConfig->getConfigData( ++ ConfigOptionsListConstants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION ++ )) { ++ return $this->templateMinifier->getMinified($template); ++ } ++ return $this->templateMinifier->getPathToMinified($template); ++ } + } diff --git a/patches/MAGECLOUD-1601__trim_static_content_path__2.1.4.patch b/patches/MAGECLOUD-1601__trim_static_content_path__2.1.4.patch new file mode 100644 index 0000000..759df9e --- /dev/null +++ b/patches/MAGECLOUD-1601__trim_static_content_path__2.1.4.patch @@ -0,0 +1,11 @@ +diff -Naur a/pub/front-static.php b/pub/front-static.php +--- a/pub/front-static.php ++++ b/pub/front-static.php +@@ -6,6 +6,7 @@ + * See COPYING.txt for license details. + */ + ++$_GET['resource'] = preg_replace('/^(\/static\/)(version(\d+)?\/)?|(\?.*)/', '', $_SERVER['REQUEST_URI'] ?: ''); + require realpath(__DIR__) . '/../app/bootstrap.php'; + $bootstrap = \Magento\Framework\App\Bootstrap::create(BP, $_SERVER); + /** @var \Magento\Framework\App\StaticResource $app */ diff --git a/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.13.patch b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.13.patch new file mode 100644 index 0000000..48905b1 --- /dev/null +++ b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.13.patch @@ -0,0 +1,907 @@ +diff -Naur a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +--- a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php ++++ b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +@@ -9,9 +9,12 @@ + */ + namespace Magento\Cron\Observer; + +-use Magento\Framework\Console\CLI; ++use Magento\Framework\App\State; ++use Magento\Framework\Console\Cli; + use Magento\Framework\Event\ObserverInterface; + use \Magento\Cron\Model\Schedule; ++use Magento\Framework\Profiler\Driver\Standard\Stat; ++use Magento\Framework\Profiler\Driver\Standard\StatFactory; + + /** + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) +@@ -96,9 +99,9 @@ class ProcessCronQueueObserver implements ObserverInterface + protected $_shell; + + /** +- * @var \Magento\Framework\Stdlib\DateTime\TimezoneInterface ++ * @var \Magento\Framework\Stdlib\DateTime\DateTime + */ +- protected $timezone; ++ protected $dateTime; + + /** + * @var \Symfony\Component\Process\PhpExecutableFinder +@@ -106,15 +109,44 @@ class ProcessCronQueueObserver implements ObserverInterface + protected $phpExecutableFinder; + + /** ++ * @var \Psr\Log\LoggerInterface ++ */ ++ private $logger; ++ ++ /** ++ * @var \Magento\Framework\App\State ++ */ ++ private $state; ++ ++ /** ++ * @var array ++ */ ++ private $invalid = []; ++ ++ /** ++ * @var array ++ */ ++ private $jobs; ++ ++ /** ++ * @var Stat ++ */ ++ private $statProfiler; ++ ++ /** + * @param \Magento\Framework\ObjectManagerInterface $objectManager +- * @param ScheduleFactory $scheduleFactory ++ * @param \Magento\Cron\Model\ScheduleFactory $scheduleFactory + * @param \Magento\Framework\App\CacheInterface $cache +- * @param ConfigInterface $config ++ * @param \Magento\Cron\Model\ConfigInterface $config + * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig + * @param \Magento\Framework\App\Console\Request $request + * @param \Magento\Framework\ShellInterface $shell +- * @param \Magento\Framework\Stdlib\DateTime\TimezoneInterface $timezone ++ * @param \Magento\Framework\Stdlib\DateTime\DateTime $dateTime + * @param \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory ++ * @param \Psr\Log\LoggerInterface $logger ++ * @param \Magento\Framework\App\State $state ++ * @param StatFactory $statFactory ++ * @SuppressWarnings(PHPMD.ExcessiveParameterList) + */ + public function __construct( + \Magento\Framework\ObjectManagerInterface $objectManager, +@@ -124,8 +156,11 @@ class ProcessCronQueueObserver implements ObserverInterface + \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig, + \Magento\Framework\App\Console\Request $request, + \Magento\Framework\ShellInterface $shell, +- \Magento\Framework\Stdlib\DateTime\TimezoneInterface $timezone, +- \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory ++ \Magento\Framework\Stdlib\DateTime\DateTime $dateTime, ++ \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory, ++ \Psr\Log\LoggerInterface $logger, ++ \Magento\Framework\App\State $state, ++ StatFactory $statFactory + ) { + $this->_objectManager = $objectManager; + $this->_scheduleFactory = $scheduleFactory; +@@ -134,8 +169,11 @@ class ProcessCronQueueObserver implements ObserverInterface + $this->_scopeConfig = $scopeConfig; + $this->_request = $request; + $this->_shell = $shell; +- $this->timezone = $timezone; ++ $this->dateTime = $dateTime; + $this->phpExecutableFinder = $phpExecutableFinderFactory->create(); ++ $this->logger = $logger; ++ $this->state = $state; ++ $this->statProfiler = $statFactory->create(); + } + + /** +@@ -151,26 +189,29 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + public function execute(\Magento\Framework\Event\Observer $observer) + { +- $pendingJobs = $this->_getPendingSchedules(); +- $currentTime = $this->timezone->scopeTimeStamp(); ++ ++ $currentTime = $this->dateTime->gmtTimestamp(); + $jobGroupsRoot = $this->_config->getJobs(); ++ // sort jobs groups to start from used in separated process ++ uksort( ++ $jobGroupsRoot, ++ function ($a, $b) { ++ return $this->getCronGroupConfigurationValue($b, 'use_separate_process') ++ - $this->getCronGroupConfigurationValue($a, 'use_separate_process'); ++ } ++ ); + + $phpPath = $this->phpExecutableFinder->find() ?: 'php'; + + foreach ($jobGroupsRoot as $groupId => $jobsRoot) { +- if ($this->_request->getParam('group') !== null +- && $this->_request->getParam('group') !== '\'' . ($groupId) . '\'' +- && $this->_request->getParam('group') !== $groupId) { ++ if (!$this->isGroupInFilter($groupId)) { + continue; + } +- if (($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1') && ( +- $this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/use_separate_process', +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ) == 1 +- )) { ++ if ($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1' ++ && $this->getCronGroupConfigurationValue($groupId, 'use_separate_process') == 1 ++ ) { + $this->_shell->execute( +- $phpPath . ' %s cron:run --group=' . $groupId . ' --' . CLI::INPUT_KEY_BOOTSTRAP . '=' ++ $phpPath . ' %s cron:run --group=' . $groupId . ' --' . Cli::INPUT_KEY_BOOTSTRAP . '=' + . self::STANDALONE_PROCESS_STARTED . '=1', + [ + BP . '/bin/magento' +@@ -179,29 +220,9 @@ class ProcessCronQueueObserver implements ObserverInterface + continue; + } + +- foreach ($pendingJobs as $schedule) { +- $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; +- if (!$jobConfig) { +- continue; +- } +- +- $scheduledTime = strtotime($schedule->getScheduledAt()); +- if ($scheduledTime > $currentTime) { +- continue; +- } +- +- try { +- if ($schedule->tryLockJob()) { +- $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); +- } +- } catch (\Exception $e) { +- $schedule->setMessages($e->getMessage()); +- } +- $schedule->save(); +- } +- +- $this->_generate($groupId); +- $this->_cleanup($groupId); ++ $this->cleanupJobs($groupId, $currentTime); ++ $this->generateSchedules($groupId); ++ $this->processPendingJobs($groupId, $jobsRoot, $currentTime); + } + } + +@@ -218,58 +239,105 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function _runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId) + { +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $jobCode = $schedule->getJobCode(); ++ $scheduleLifetime = $this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_LIFETIME); + $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + if ($scheduledTime < $currentTime - $scheduleLifetime) { + $schedule->setStatus(Schedule::STATUS_MISSED); ++ $this->logger->info(sprintf('Cron Job %s is missed', $jobCode)); + throw new \Exception('Too late for the schedule'); + } + + if (!isset($jobConfig['instance'], $jobConfig['method'])) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception('No callbacks found'); + } + $model = $this->_objectManager->create($jobConfig['instance']); + $callback = [$model, $jobConfig['method']]; + if (!is_callable($callback)) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception( + sprintf('Invalid callback: %s::%s can\'t be called', $jobConfig['instance'], $jobConfig['method']) + ); + } + +- $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->timezone->scopeTimeStamp()))->save(); ++ $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp()))->save(); + ++ $this->startProfiling(); + try { ++ $this->logger->info(sprintf('Cron Job %s is run', $jobCode)); + call_user_func_array($callback, [$schedule]); + } catch (\Exception $e) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf( ++ 'Cron Job %s has an error. Statistics: %s %s', ++ $jobCode, ++ $this->getProfilingStat(), $e->getMessage() ++ )); + throw $e; ++ } finally { ++ $this->stopProfiling(); + } + + $schedule->setStatus(Schedule::STATUS_SUCCESS)->setFinishedAt(strftime( + '%Y-%m-%d %H:%M:%S', +- $this->timezone->scopeTimeStamp() ++ $this->dateTime->gmtTimestamp() ++ )); ++ ++ $this->logger->info(sprintf( ++ 'Cron Job %s is successfully finished. Statistics: %s', ++ $jobCode, ++ $this->getProfilingStat() + )); + } + + /** ++ * Starts profiling ++ * ++ * @return void ++ */ ++ private function startProfiling() ++ { ++ $this->statProfiler->clear(); ++ $this->statProfiler->start('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Stops profiling ++ * ++ * @return void ++ */ ++ private function stopProfiling() ++ { ++ $this->statProfiler->stop('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Retrieves statistics in the JSON format ++ * ++ * @return string ++ */ ++ private function getProfilingStat() ++ { ++ $stat = $this->statProfiler->get('job'); ++ unset($stat[Stat::START]); ++ return json_encode($stat); ++ } ++ ++ /** + * Return job collection from data base with status 'pending' + * + * @return \Magento\Cron\Model\ResourceModel\Schedule\Collection + */ +- protected function _getPendingSchedules() ++ private function getPendingSchedules($groupId) + { +- if (!$this->_pendingSchedules) { +- $this->_pendingSchedules = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- Schedule::STATUS_PENDING +- )->load(); +- } +- return $this->_pendingSchedules; ++ $jobs = $this->getJobs(); ++ $pendingJobs = $this->_scheduleFactory->create()->getCollection(); ++ $pendingJobs->addFieldToFilter('status', Schedule::STATUS_PENDING); ++ $pendingJobs->addFieldToFilter('job_code', ['in' => array_keys($jobs[$groupId])]); ++ return $pendingJobs; + } + + /** +@@ -278,22 +346,32 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param string $groupId + * @return $this + */ +- protected function _generate($groupId) ++ private function generateSchedules($groupId) + { + /** + * check if schedule generation is needed + */ + $lastRun = (int)$this->_cache->load(self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId); +- $rawSchedulePeriod = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_GENERATE_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ $rawSchedulePeriod = (int)$this->getCronGroupConfigurationValue( ++ $groupId, ++ self::XML_PATH_SCHEDULE_GENERATE_EVERY + ); + $schedulePeriod = $rawSchedulePeriod * self::SECONDS_IN_MINUTE; +- if ($lastRun > $this->timezone->scopeTimeStamp() - $schedulePeriod) { ++ if ($lastRun > $this->dateTime->gmtTimestamp() - $schedulePeriod) { + return $this; + } + +- $schedules = $this->_getPendingSchedules(); ++ /** ++ * save time schedules generation was ran with no expiration ++ */ ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, ++ ['crontab'], ++ null ++ ); ++ ++ $schedules = $this->getPendingSchedules($groupId); + $exists = []; + /** @var Schedule $schedule */ + foreach ($schedules as $schedule) { +@@ -303,18 +381,10 @@ class ProcessCronQueueObserver implements ObserverInterface + /** + * generate global crontab jobs + */ +- $jobs = $this->_config->getJobs(); ++ $jobs = $this->getJobs(); ++ $this->invalid = []; + $this->_generateJobs($jobs[$groupId], $exists, $groupId); +- +- /** +- * save time schedules generation was ran with no expiration +- */ +- $this->_cache->save( +- $this->timezone->scopeTimeStamp(), +- self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, +- ['crontab'], +- null +- ); ++ $this->cleanupScheduleMismatches(); + + return $this; + } +@@ -325,22 +395,12 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param array $jobs + * @param array $exists + * @param string $groupId +- * @return $this ++ * @return void + */ + protected function _generateJobs($jobs, $exists, $groupId) + { + foreach ($jobs as $jobCode => $jobConfig) { +- $cronExpression = null; +- if (isset($jobConfig['config_path'])) { +- $cronExpression = $this->getConfigSchedule($jobConfig) ?: null; +- } +- +- if (!$cronExpression) { +- if (isset($jobConfig['schedule'])) { +- $cronExpression = $jobConfig['schedule']; +- } +- } +- ++ $cronExpression = $this->getCronExpression($jobConfig); + if (!$cronExpression) { + continue; + } +@@ -348,75 +408,60 @@ class ProcessCronQueueObserver implements ObserverInterface + $timeInterval = $this->getScheduleTimeInterval($groupId); + $this->saveSchedule($jobCode, $cronExpression, $timeInterval, $exists); + } +- return $this; + } + + /** +- * Clean existed jobs ++ * Clean expired jobs + * +- * @param string $groupId +- * @return $this ++ * @param $groupId ++ * @param $currentTime ++ * @return void + */ +- protected function _cleanup($groupId) ++ private function cleanupJobs($groupId, $currentTime) + { + // check if history cleanup is needed + $lastCleanup = (int)$this->_cache->load(self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId); +- $historyCleanUp = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_CLEANUP_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); +- if ($lastCleanup > $this->timezone->scopeTimeStamp() - $historyCleanUp * self::SECONDS_IN_MINUTE) { ++ $historyCleanUp = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_CLEANUP_EVERY); ++ if ($lastCleanup > $this->dateTime->gmtTimestamp() - $historyCleanUp * self::SECONDS_IN_MINUTE) { + return $this; + } +- +- // check how long the record should stay unprocessed before marked as MISSED +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ // save time history cleanup was ran with no expiration ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, ++ ['crontab'], ++ null + ); +- $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + +- /** +- * @var \Magento\Cron\Model\ResourceModel\Schedule\Collection $history +- */ +- $history = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- ['in' => [Schedule::STATUS_SUCCESS, Schedule::STATUS_MISSED, Schedule::STATUS_ERROR]] +- )->load(); ++ $this->cleanupDisabledJobs($groupId); + +- $historySuccess = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_SUCCESS, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); +- $historyFailure = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_FAILURE, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $historySuccess = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_SUCCESS); ++ $historyFailure = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_FAILURE); + $historyLifetimes = [ + Schedule::STATUS_SUCCESS => $historySuccess * self::SECONDS_IN_MINUTE, + Schedule::STATUS_MISSED => $historyFailure * self::SECONDS_IN_MINUTE, + Schedule::STATUS_ERROR => $historyFailure * self::SECONDS_IN_MINUTE, ++ Schedule::STATUS_PENDING => max($historyFailure, $historySuccess) * self::SECONDS_IN_MINUTE, + ]; + +- $now = $this->timezone->scopeTimeStamp(); +- /** @var Schedule $record */ +- foreach ($history as $record) { +- $checkTime = $record->getExecutedAt() ? strtotime($record->getExecutedAt()) : +- strtotime($record->getScheduledAt()) + $scheduleLifetime; +- if ($checkTime < $now - $historyLifetimes[$record->getStatus()]) { +- $record->delete(); +- } ++ $jobs = $this->getJobs()[$groupId]; ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $connection = $scheduleResource->getConnection(); ++ $count = 0; ++ foreach ($historyLifetimes as $status => $time) { ++ $count += $connection->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => $status, ++ 'job_code in (?)' => array_keys($jobs), ++ 'created_at < ?' => $connection->formatDate($currentTime - $time) ++ ] ++ ); + } + +- // save time history cleanup was ran with no expiration +- $this->_cache->save( +- $this->timezone->scopeTimeStamp(), +- self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, +- ['crontab'], +- null +- ); +- +- return $this; ++ if ($count) { ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } + } + + /** +@@ -442,19 +487,25 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function saveSchedule($jobCode, $cronExpression, $timeInterval, $exists) + { +- $currentTime = $this->timezone->scopeTimeStamp(); ++ $currentTime = $this->dateTime->gmtTimestamp(); + $timeAhead = $currentTime + $timeInterval; + for ($time = $currentTime; $time < $timeAhead; $time += self::SECONDS_IN_MINUTE) { +- $ts = strftime('%Y-%m-%d %H:%M:00', $time); +- if (!empty($exists[$jobCode . '/' . $ts])) { +- // already scheduled ++ $scheduledAt = strftime('%Y-%m-%d %H:%M:00', $time); ++ $alreadyScheduled = !empty($exists[$jobCode . '/' . $scheduledAt]); ++ $schedule = $this->createSchedule($jobCode, $cronExpression, $time); ++ $valid = $schedule->trySchedule(); ++ if (!$valid) { ++ if ($alreadyScheduled) { ++ if (!isset($this->invalid[$jobCode])) { ++ $this->invalid[$jobCode] = []; ++ } ++ $this->invalid[$jobCode][] = $scheduledAt; ++ } + continue; + } +- $schedule = $this->generateSchedule($jobCode, $cronExpression, $time); +- if ($schedule->trySchedule()) { ++ if (!$alreadyScheduled) { + // time matches cron expression + $schedule->save(); +- return; + } + } + } +@@ -465,13 +516,13 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param int $time + * @return Schedule + */ +- protected function generateSchedule($jobCode, $cronExpression, $time) ++ protected function createSchedule($jobCode, $cronExpression, $time) + { + $schedule = $this->_scheduleFactory->create() + ->setCronExpr($cronExpression) + ->setJobCode($jobCode) + ->setStatus(Schedule::STATUS_PENDING) +- ->setCreatedAt(strftime('%Y-%m-%d %H:%M:%S', $this->timezone->scopeTimeStamp())) ++ ->setCreatedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp())) + ->setScheduledAt(strftime('%Y-%m-%d %H:%M', $time)); + + return $schedule; +@@ -483,12 +534,174 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function getScheduleTimeInterval($groupId) + { +- $scheduleAheadFor = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_AHEAD_FOR, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $scheduleAheadFor = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_AHEAD_FOR); + $scheduleAheadFor = $scheduleAheadFor * self::SECONDS_IN_MINUTE; + + return $scheduleAheadFor; + } ++ ++ /** ++ * Clean up scheduled jobs that are disabled in the configuration ++ * This can happen when you turn off a cron job in the config and flush the cache ++ * ++ * @param string $groupId ++ * @return void ++ */ ++ private function cleanupDisabledJobs($groupId) ++ { ++ $jobs = $this->getJobs(); ++ $jobsToCleanup = []; ++ foreach ($jobs[$groupId] as $jobCode => $jobConfig) { ++ if (!$this->getCronExpression($jobConfig)) { ++ /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ ++ $jobsToCleanup[] = $jobCode; ++ } ++ } ++ ++ if (count($jobsToCleanup) > 0) { ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $count = $scheduleResource->getConnection()->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code in (?)' => $jobsToCleanup, ++ ] ++ ); ++ ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } ++ } ++ ++ /** ++ * @param array $jobConfig ++ * @return null|string ++ */ ++ private function getCronExpression($jobConfig) ++ { ++ $cronExpression = null; ++ if (isset($jobConfig['config_path'])) { ++ $cronExpression = $this->getConfigSchedule($jobConfig) ?: null; ++ } ++ ++ if (!$cronExpression) { ++ if (isset($jobConfig['schedule'])) { ++ $cronExpression = $jobConfig['schedule']; ++ } ++ } ++ return $cronExpression; ++ } ++ ++ /** ++ * Clean up scheduled jobs that do not match their cron expression anymore ++ * This can happen when you change the cron expression and flush the cache ++ * ++ * @return $this ++ */ ++ private function cleanupScheduleMismatches() ++ { ++ /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ foreach ($this->invalid as $jobCode => $scheduledAtList) { ++ $scheduleResource->getConnection()->delete($scheduleResource->getMainTable(), [ ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code = ?' => $jobCode, ++ 'scheduled_at in (?)' => $scheduledAtList, ++ ]); ++ } ++ return $this; ++ } ++ ++ /** ++ * @return array ++ */ ++ private function getJobs() ++ { ++ if ($this->jobs === null) { ++ $this->jobs = $this->_config->getJobs(); ++ } ++ return $this->jobs; ++ } ++ ++ /** ++ * Get CronGroup Configuration Value ++ * ++ * @param $groupId ++ * @return int ++ */ ++ private function getCronGroupConfigurationValue($groupId, $path) ++ { ++ return $this->_scopeConfig->getValue( ++ 'system/cron/' . $groupId . '/' . $path, ++ \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ ); ++ return $scheduleLifetime; ++ } ++ ++ /** ++ * Is Group In Filter ++ * ++ * @param $groupId ++ * @return bool ++ */ ++ private function isGroupInFilter($groupId): bool ++ { ++ return !($this->_request->getParam('group') !== null ++ && trim($this->_request->getParam('group'), "'") !== $groupId); ++ } ++ ++ /** ++ * Process pending jobs ++ * ++ * @param $groupId ++ * @param $jobsRoot ++ * @param $currentTime ++ */ ++ private function processPendingJobs($groupId, $jobsRoot, $currentTime) ++ { ++ $procesedJobs = []; ++ $pendingJobs = $this->getPendingSchedules($groupId); ++ /** @var \Magento\Cron\Model\Schedule $schedule */ ++ foreach ($pendingJobs as $schedule) { ++ if (isset($procesedJobs[$schedule->getJobCode()])) { ++ // process only on job per run ++ continue; ++ } ++ $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; ++ if (!$jobConfig) { ++ continue; ++ } ++ ++ $scheduledTime = strtotime($schedule->getScheduledAt()); ++ if ($scheduledTime > $currentTime) { ++ continue; ++ } ++ ++ try { ++ if ($schedule->tryLockJob()) { ++ $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); ++ } ++ } catch (\Exception $e) { ++ $schedule->setMessages($e->getMessage()); ++ if ($schedule->getStatus() === Schedule::STATUS_ERROR) { ++ $this->logger->critical($e); ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_MISSED ++ && $this->state->getMode() === State::MODE_DEVELOPER ++ ) { ++ $this->logger->error( ++ sprintf( ++ "%s Schedule Id: %s Job Code: %s", ++ $schedule->getMessages(), ++ $schedule->getScheduleId(), ++ $schedule->getJobCode() ++ ) ++ ); ++ } ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_SUCCESS) { ++ $procesedJobs[$schedule->getJobCode()] = true; ++ } ++ $schedule->save(); ++ } ++ } + } + +diff -Naur a/vendor/magento/module-cron/Model/Schedule.php b/vendor/magento/module-cron/Model/Schedule.php +--- a/vendor/magento/module-cron/Model/Schedule.php ++++ b/vendor/magento/module-cron/Model/Schedule.php +@@ -1,18 +1,18 @@ + ++ * @api ++ * @since 100.0.2 + */ + class Schedule extends \Magento\Framework\Model\AbstractModel + { +@@ -45,20 +46,28 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + const STATUS_ERROR = 'error'; + + /** ++ * @var TimezoneInterface ++ */ ++ private $timezoneConverter; ++ ++ /** + * @param \Magento\Framework\Model\Context $context + * @param \Magento\Framework\Registry $registry + * @param \Magento\Framework\Model\ResourceModel\AbstractResource $resource + * @param \Magento\Framework\Data\Collection\AbstractDb $resourceCollection + * @param array $data ++ * @param TimezoneInterface $timezoneConverter + */ + public function __construct( + \Magento\Framework\Model\Context $context, + \Magento\Framework\Registry $registry, + \Magento\Framework\Model\ResourceModel\AbstractResource $resource = null, + \Magento\Framework\Data\Collection\AbstractDb $resourceCollection = null, +- array $data = [] ++ array $data = [], ++ TimezoneInterface $timezoneConverter = null + ) { + parent::__construct($context, $registry, $resource, $resourceCollection, $data); ++ $this->timezoneConverter = $timezoneConverter ?: ObjectManager::getInstance()->get(TimezoneInterface::class); + } + + /** +@@ -66,7 +75,7 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + */ + public function _construct() + { +- $this->_init('Magento\Cron\Model\ResourceModel\Schedule'); ++ $this->_init(\Magento\Cron\Model\ResourceModel\Schedule::class); + } + + /** +@@ -101,6 +110,9 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + return false; + } + if (!is_numeric($time)) { ++ //convert time from UTC to admin store timezone ++ //we assume that all schedules in configuration (crontab.xml and DB tables) are in admin store timezone ++ $time = $this->timezoneConverter->date($time)->format('Y-m-d H:i'); + $time = strtotime($time); + } + $match = $this->matchCronExpression($e[0], strftime('%M', $time)) +@@ -221,16 +233,17 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + } + + /** +- * Sets a job to STATUS_RUNNING only if it is currently in STATUS_PENDING. +- * Returns true if status was changed and false otherwise. ++ * Lock the cron job so no other scheduled instances run simultaneously. + * +- * This is used to implement locking for cron jobs. ++ * Sets a job to STATUS_RUNNING only if it is currently in STATUS_PENDING ++ * and no other jobs of the same code are currently in STATUS_RUNNING. ++ * Returns true if status was changed and false otherwise. + * + * @return boolean + */ + public function tryLockJob() + { +- if ($this->_getResource()->trySetJobStatusAtomic( ++ if ($this->_getResource()->trySetJobUniqueStatusAtomic( + $this->getId(), + self::STATUS_RUNNING, + self::STATUS_PENDING + +diff -Naur a/vendor/magento/module-cron/Model/ResourceModel/Schedule.php b/vendor/magento/module-cron/Model/ResourceModel/Schedule.php +index dca4e22..25dd02c 100644 +--- a/vendor/magento/module-cron/Model/ResourceModel/Schedule.php ++++ b/vendor/magento/module-cron/Model/ResourceModel/Schedule.php +@@ -8,7 +8,8 @@ namespace Magento\Cron\Model\ResourceModel; + /** + * Schedule resource + * +- * @author Magento Core Team ++ * @api ++ * @since 100.0.2 + */ + class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + { +@@ -23,9 +24,10 @@ class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + } + + /** +- * If job is currently in $currentStatus, set it to $newStatus +- * and return true. Otherwise, return false and do not change the job. +- * This method is used to implement locking for cron jobs. ++ * Sets new schedule status only if it's in the expected current status. ++ * ++ * If schedule is currently in $currentStatus, set it to $newStatus and ++ * return true. Otherwise, return false. + * + * @param string $scheduleId + * @param string $newStatus +@@ -45,4 +47,49 @@ class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + } + return false; + } ++ ++ /** ++ * Sets schedule status only if no existing schedules with the same job code ++ * have that status. This is used to implement locking for cron jobs. ++ * ++ * If the schedule is currently in $currentStatus and there are no existing ++ * schedules with the same job code and $newStatus, set the schedule to ++ * $newStatus and return true. Otherwise, return false. ++ * ++ * @param string $scheduleId ++ * @param string $newStatus ++ * @param string $currentStatus ++ * @return bool ++ * @since 100.2.0 ++ */ ++ public function trySetJobUniqueStatusAtomic($scheduleId, $newStatus, $currentStatus) ++ { ++ $connection = $this->getConnection(); ++ ++ // this condition added to avoid cron jobs locking after incorrect termination of running job ++ $match = $connection->quoteInto( ++ 'existing.job_code = current.job_code ' . ++ 'AND (existing.executed_at > UTC_TIMESTAMP() - INTERVAL 1 DAY OR existing.executed_at IS NULL) ' . ++ 'AND existing.status = ?', ++ $newStatus ++ ); ++ ++ $selectIfUnlocked = $connection->select() ++ ->joinLeft( ++ ['existing' => $this->getTable('cron_schedule')], ++ $match, ++ ['status' => new \Zend_Db_Expr($connection->quote($newStatus))] ++ ) ++ ->where('current.schedule_id = ?', $scheduleId) ++ ->where('current.status = ?', $currentStatus) ++ ->where('existing.schedule_id IS NULL'); ++ ++ $update = $connection->updateFromSelect($selectIfUnlocked, ['current' => $this->getTable('cron_schedule')]); ++ $result = $connection->query($update)->rowCount(); ++ ++ if ($result == 1) { ++ return true; ++ } ++ return false; ++ } + } diff --git a/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.14.patch b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.14.patch new file mode 100644 index 0000000..117d4d7 --- /dev/null +++ b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.14.patch @@ -0,0 +1,902 @@ +diff -Naur a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +--- a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php ++++ b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +@@ -9,9 +9,12 @@ + */ + namespace Magento\Cron\Observer; + ++use Magento\Framework\App\State; + use Magento\Framework\Console\Cli; + use Magento\Framework\Event\ObserverInterface; + use \Magento\Cron\Model\Schedule; ++use Magento\Framework\Profiler\Driver\Standard\Stat; ++use Magento\Framework\Profiler\Driver\Standard\StatFactory; + + /** + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) +@@ -96,25 +99,54 @@ class ProcessCronQueueObserver implements ObserverInterface + protected $_shell; + + /** +- * @var \Magento\Framework\Stdlib\DateTime\TimezoneInterface ++ * @var \Magento\Framework\Stdlib\DateTime\DateTime + */ +- protected $timezone; ++ protected $dateTime; + + /** + * @var \Symfony\Component\Process\PhpExecutableFinder + */ + protected $phpExecutableFinder; + ++ /** ++ * @var \Psr\Log\LoggerInterface ++ */ ++ private $logger; ++ ++ /** ++ * @var \Magento\Framework\App\State ++ */ ++ private $state; ++ ++ /** ++ * @var array ++ */ ++ private $invalid = []; ++ ++ /** ++ * @var array ++ */ ++ private $jobs; ++ ++ /** ++ * @var Stat ++ */ ++ private $statProfiler; ++ + /** + * @param \Magento\Framework\ObjectManagerInterface $objectManager +- * @param ScheduleFactory $scheduleFactory ++ * @param \Magento\Cron\Model\ScheduleFactory $scheduleFactory + * @param \Magento\Framework\App\CacheInterface $cache +- * @param ConfigInterface $config ++ * @param \Magento\Cron\Model\ConfigInterface $config + * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig + * @param \Magento\Framework\App\Console\Request $request + * @param \Magento\Framework\ShellInterface $shell +- * @param \Magento\Framework\Stdlib\DateTime\TimezoneInterface $timezone ++ * @param \Magento\Framework\Stdlib\DateTime\DateTime $dateTime + * @param \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory ++ * @param \Psr\Log\LoggerInterface $logger ++ * @param \Magento\Framework\App\State $state ++ * @param StatFactory $statFactory ++ * @SuppressWarnings(PHPMD.ExcessiveParameterList) + */ + public function __construct( + \Magento\Framework\ObjectManagerInterface $objectManager, +@@ -124,8 +156,11 @@ class ProcessCronQueueObserver implements ObserverInterface + \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig, + \Magento\Framework\App\Console\Request $request, + \Magento\Framework\ShellInterface $shell, +- \Magento\Framework\Stdlib\DateTime\TimezoneInterface $timezone, +- \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory ++ \Magento\Framework\Stdlib\DateTime\DateTime $dateTime, ++ \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory, ++ \Psr\Log\LoggerInterface $logger, ++ \Magento\Framework\App\State $state, ++ StatFactory $statFactory + ) { + $this->_objectManager = $objectManager; + $this->_scheduleFactory = $scheduleFactory; +@@ -134,8 +169,11 @@ class ProcessCronQueueObserver implements ObserverInterface + $this->_scopeConfig = $scopeConfig; + $this->_request = $request; + $this->_shell = $shell; +- $this->timezone = $timezone; ++ $this->dateTime = $dateTime; + $this->phpExecutableFinder = $phpExecutableFinderFactory->create(); ++ $this->logger = $logger; ++ $this->state = $state; ++ $this->statProfiler = $statFactory->create(); + } + + /** +@@ -151,26 +189,29 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + public function execute(\Magento\Framework\Event\Observer $observer) + { +- $pendingJobs = $this->_getPendingSchedules(); +- $currentTime = $this->timezone->scopeTimeStamp(); ++ ++ $currentTime = $this->dateTime->gmtTimestamp(); + $jobGroupsRoot = $this->_config->getJobs(); ++ // sort jobs groups to start from used in separated process ++ uksort( ++ $jobGroupsRoot, ++ function ($a, $b) { ++ return $this->getCronGroupConfigurationValue($b, 'use_separate_process') ++ - $this->getCronGroupConfigurationValue($a, 'use_separate_process'); ++ } ++ ); + + $phpPath = $this->phpExecutableFinder->find() ?: 'php'; + + foreach ($jobGroupsRoot as $groupId => $jobsRoot) { +- if ($this->_request->getParam('group') !== null +- && $this->_request->getParam('group') !== '\'' . ($groupId) . '\'' +- && $this->_request->getParam('group') !== $groupId) { ++ if (!$this->isGroupInFilter($groupId)) { + continue; + } +- if (($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1') && ( +- $this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/use_separate_process', +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ) == 1 +- )) { ++ if ($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1' ++ && $this->getCronGroupConfigurationValue($groupId, 'use_separate_process') == 1 ++ ) { + $this->_shell->execute( + $phpPath . ' %s cron:run --group=' . $groupId . ' --' . Cli::INPUT_KEY_BOOTSTRAP . '=' + . self::STANDALONE_PROCESS_STARTED . '=1', + [ + BP . '/bin/magento' +@@ -179,29 +220,9 @@ class ProcessCronQueueObserver implements ObserverInterface + continue; + } + +- foreach ($pendingJobs as $schedule) { +- $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; +- if (!$jobConfig) { +- continue; +- } +- +- $scheduledTime = strtotime($schedule->getScheduledAt()); +- if ($scheduledTime > $currentTime) { +- continue; +- } +- +- try { +- if ($schedule->tryLockJob()) { +- $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); +- } +- } catch (\Exception $e) { +- $schedule->setMessages($e->getMessage()); +- } +- $schedule->save(); +- } +- +- $this->_generate($groupId); +- $this->_cleanup($groupId); ++ $this->cleanupJobs($groupId, $currentTime); ++ $this->generateSchedules($groupId); ++ $this->processPendingJobs($groupId, $jobsRoot, $currentTime); + } + } + +@@ -218,58 +239,105 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function _runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId) + { +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $jobCode = $schedule->getJobCode(); ++ $scheduleLifetime = $this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_LIFETIME); + $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + if ($scheduledTime < $currentTime - $scheduleLifetime) { + $schedule->setStatus(Schedule::STATUS_MISSED); ++ $this->logger->info(sprintf('Cron Job %s is missed', $jobCode)); + throw new \Exception('Too late for the schedule'); + } + + if (!isset($jobConfig['instance'], $jobConfig['method'])) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception('No callbacks found'); + } + $model = $this->_objectManager->create($jobConfig['instance']); + $callback = [$model, $jobConfig['method']]; + if (!is_callable($callback)) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception( + sprintf('Invalid callback: %s::%s can\'t be called', $jobConfig['instance'], $jobConfig['method']) + ); + } + +- $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->timezone->scopeTimeStamp()))->save(); ++ $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp()))->save(); + ++ $this->startProfiling(); + try { ++ $this->logger->info(sprintf('Cron Job %s is run', $jobCode)); + call_user_func_array($callback, [$schedule]); + } catch (\Exception $e) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf( ++ 'Cron Job %s has an error. Statistics: %s %s', ++ $jobCode, ++ $this->getProfilingStat(), $e->getMessage() ++ )); + throw $e; ++ } finally { ++ $this->stopProfiling(); + } + + $schedule->setStatus(Schedule::STATUS_SUCCESS)->setFinishedAt(strftime( + '%Y-%m-%d %H:%M:%S', +- $this->timezone->scopeTimeStamp() ++ $this->dateTime->gmtTimestamp() ++ )); ++ ++ $this->logger->info(sprintf( ++ 'Cron Job %s is successfully finished. Statistics: %s', ++ $jobCode, ++ $this->getProfilingStat() + )); + } + ++ /** ++ * Starts profiling ++ * ++ * @return void ++ */ ++ private function startProfiling() ++ { ++ $this->statProfiler->clear(); ++ $this->statProfiler->start('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Stops profiling ++ * ++ * @return void ++ */ ++ private function stopProfiling() ++ { ++ $this->statProfiler->stop('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Retrieves statistics in the JSON format ++ * ++ * @return string ++ */ ++ private function getProfilingStat() ++ { ++ $stat = $this->statProfiler->get('job'); ++ unset($stat[Stat::START]); ++ return json_encode($stat); ++ } ++ + /** + * Return job collection from data base with status 'pending' + * + * @return \Magento\Cron\Model\ResourceModel\Schedule\Collection + */ +- protected function _getPendingSchedules() ++ private function getPendingSchedules($groupId) + { +- if (!$this->_pendingSchedules) { +- $this->_pendingSchedules = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- Schedule::STATUS_PENDING +- )->load(); +- } +- return $this->_pendingSchedules; ++ $jobs = $this->getJobs(); ++ $pendingJobs = $this->_scheduleFactory->create()->getCollection(); ++ $pendingJobs->addFieldToFilter('status', Schedule::STATUS_PENDING); ++ $pendingJobs->addFieldToFilter('job_code', ['in' => array_keys($jobs[$groupId])]); ++ return $pendingJobs; + } + + /** +@@ -278,22 +346,32 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param string $groupId + * @return $this + */ +- protected function _generate($groupId) ++ private function generateSchedules($groupId) + { + /** + * check if schedule generation is needed + */ + $lastRun = (int)$this->_cache->load(self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId); +- $rawSchedulePeriod = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_GENERATE_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ $rawSchedulePeriod = (int)$this->getCronGroupConfigurationValue( ++ $groupId, ++ self::XML_PATH_SCHEDULE_GENERATE_EVERY + ); + $schedulePeriod = $rawSchedulePeriod * self::SECONDS_IN_MINUTE; +- if ($lastRun > $this->timezone->scopeTimeStamp() - $schedulePeriod) { ++ if ($lastRun > $this->dateTime->gmtTimestamp() - $schedulePeriod) { + return $this; + } + +- $schedules = $this->_getPendingSchedules(); ++ /** ++ * save time schedules generation was ran with no expiration ++ */ ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, ++ ['crontab'], ++ null ++ ); ++ ++ $schedules = $this->getPendingSchedules($groupId); + $exists = []; + /** @var Schedule $schedule */ + foreach ($schedules as $schedule) { +@@ -303,18 +381,10 @@ class ProcessCronQueueObserver implements ObserverInterface + /** + * generate global crontab jobs + */ +- $jobs = $this->_config->getJobs(); ++ $jobs = $this->getJobs(); ++ $this->invalid = []; + $this->_generateJobs($jobs[$groupId], $exists, $groupId); +- +- /** +- * save time schedules generation was ran with no expiration +- */ +- $this->_cache->save( +- $this->timezone->scopeTimeStamp(), +- self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, +- ['crontab'], +- null +- ); ++ $this->cleanupScheduleMismatches(); + + return $this; + } +@@ -325,22 +395,12 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param array $jobs + * @param array $exists + * @param string $groupId +- * @return $this ++ * @return void + */ + protected function _generateJobs($jobs, $exists, $groupId) + { + foreach ($jobs as $jobCode => $jobConfig) { +- $cronExpression = null; +- if (isset($jobConfig['config_path'])) { +- $cronExpression = $this->getConfigSchedule($jobConfig) ?: null; +- } +- +- if (!$cronExpression) { +- if (isset($jobConfig['schedule'])) { +- $cronExpression = $jobConfig['schedule']; +- } +- } +- ++ $cronExpression = $this->getCronExpression($jobConfig); + if (!$cronExpression) { + continue; + } +@@ -348,75 +408,60 @@ class ProcessCronQueueObserver implements ObserverInterface + $timeInterval = $this->getScheduleTimeInterval($groupId); + $this->saveSchedule($jobCode, $cronExpression, $timeInterval, $exists); + } +- return $this; + } + + /** +- * Clean existed jobs ++ * Clean expired jobs + * +- * @param string $groupId +- * @return $this ++ * @param $groupId ++ * @param $currentTime ++ * @return void + */ +- protected function _cleanup($groupId) ++ private function cleanupJobs($groupId, $currentTime) + { + // check if history cleanup is needed + $lastCleanup = (int)$this->_cache->load(self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId); +- $historyCleanUp = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_CLEANUP_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); +- if ($lastCleanup > $this->timezone->scopeTimeStamp() - $historyCleanUp * self::SECONDS_IN_MINUTE) { ++ $historyCleanUp = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_CLEANUP_EVERY); ++ if ($lastCleanup > $this->dateTime->gmtTimestamp() - $historyCleanUp * self::SECONDS_IN_MINUTE) { + return $this; + } +- +- // check how long the record should stay unprocessed before marked as MISSED +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ // save time history cleanup was ran with no expiration ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, ++ ['crontab'], ++ null + ); +- $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + +- /** +- * @var \Magento\Cron\Model\ResourceModel\Schedule\Collection $history +- */ +- $history = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- ['in' => [Schedule::STATUS_SUCCESS, Schedule::STATUS_MISSED, Schedule::STATUS_ERROR]] +- )->load(); ++ $this->cleanupDisabledJobs($groupId); + +- $historySuccess = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_SUCCESS, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); +- $historyFailure = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_FAILURE, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $historySuccess = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_SUCCESS); ++ $historyFailure = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_FAILURE); + $historyLifetimes = [ + Schedule::STATUS_SUCCESS => $historySuccess * self::SECONDS_IN_MINUTE, + Schedule::STATUS_MISSED => $historyFailure * self::SECONDS_IN_MINUTE, + Schedule::STATUS_ERROR => $historyFailure * self::SECONDS_IN_MINUTE, ++ Schedule::STATUS_PENDING => max($historyFailure, $historySuccess) * self::SECONDS_IN_MINUTE, + ]; + +- $now = $this->timezone->scopeTimeStamp(); +- /** @var Schedule $record */ +- foreach ($history as $record) { +- $checkTime = $record->getExecutedAt() ? strtotime($record->getExecutedAt()) : +- strtotime($record->getScheduledAt()) + $scheduleLifetime; +- if ($checkTime < $now - $historyLifetimes[$record->getStatus()]) { +- $record->delete(); +- } ++ $jobs = $this->getJobs()[$groupId]; ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $connection = $scheduleResource->getConnection(); ++ $count = 0; ++ foreach ($historyLifetimes as $status => $time) { ++ $count += $connection->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => $status, ++ 'job_code in (?)' => array_keys($jobs), ++ 'created_at < ?' => $connection->formatDate($currentTime - $time) ++ ] ++ ); + } + +- // save time history cleanup was ran with no expiration +- $this->_cache->save( +- $this->timezone->scopeTimeStamp(), +- self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, +- ['crontab'], +- null +- ); +- +- return $this; ++ if ($count) { ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } + } + + /** +@@ -442,16 +487,23 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function saveSchedule($jobCode, $cronExpression, $timeInterval, $exists) + { +- $currentTime = $this->timezone->scopeTimeStamp(); ++ $currentTime = $this->dateTime->gmtTimestamp(); + $timeAhead = $currentTime + $timeInterval; + for ($time = $currentTime; $time < $timeAhead; $time += self::SECONDS_IN_MINUTE) { +- $ts = strftime('%Y-%m-%d %H:%M:00', $time); +- if (!empty($exists[$jobCode . '/' . $ts])) { +- // already scheduled ++ $scheduledAt = strftime('%Y-%m-%d %H:%M:00', $time); ++ $alreadyScheduled = !empty($exists[$jobCode . '/' . $scheduledAt]); ++ $schedule = $this->createSchedule($jobCode, $cronExpression, $time); ++ $valid = $schedule->trySchedule(); ++ if (!$valid) { ++ if ($alreadyScheduled) { ++ if (!isset($this->invalid[$jobCode])) { ++ $this->invalid[$jobCode] = []; ++ } ++ $this->invalid[$jobCode][] = $scheduledAt; ++ } + continue; + } +- $schedule = $this->generateSchedule($jobCode, $cronExpression, $time); +- if ($schedule->trySchedule()) { ++ if (!$alreadyScheduled) { + // time matches cron expression + $schedule->save(); + } +@@ -464,13 +516,13 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param int $time + * @return Schedule + */ +- protected function generateSchedule($jobCode, $cronExpression, $time) ++ protected function createSchedule($jobCode, $cronExpression, $time) + { + $schedule = $this->_scheduleFactory->create() + ->setCronExpr($cronExpression) + ->setJobCode($jobCode) + ->setStatus(Schedule::STATUS_PENDING) +- ->setCreatedAt(strftime('%Y-%m-%d %H:%M:%S', $this->timezone->scopeTimeStamp())) ++ ->setCreatedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp())) + ->setScheduledAt(strftime('%Y-%m-%d %H:%M', $time)); + + return $schedule; +@@ -482,12 +534,174 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function getScheduleTimeInterval($groupId) + { +- $scheduleAheadFor = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_AHEAD_FOR, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $scheduleAheadFor = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_AHEAD_FOR); + $scheduleAheadFor = $scheduleAheadFor * self::SECONDS_IN_MINUTE; + + return $scheduleAheadFor; + } ++ ++ /** ++ * Clean up scheduled jobs that are disabled in the configuration ++ * This can happen when you turn off a cron job in the config and flush the cache ++ * ++ * @param string $groupId ++ * @return void ++ */ ++ private function cleanupDisabledJobs($groupId) ++ { ++ $jobs = $this->getJobs(); ++ $jobsToCleanup = []; ++ foreach ($jobs[$groupId] as $jobCode => $jobConfig) { ++ if (!$this->getCronExpression($jobConfig)) { ++ /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ ++ $jobsToCleanup[] = $jobCode; ++ } ++ } ++ ++ if (count($jobsToCleanup) > 0) { ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $count = $scheduleResource->getConnection()->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code in (?)' => $jobsToCleanup, ++ ] ++ ); ++ ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } ++ } ++ ++ /** ++ * @param array $jobConfig ++ * @return null|string ++ */ ++ private function getCronExpression($jobConfig) ++ { ++ $cronExpression = null; ++ if (isset($jobConfig['config_path'])) { ++ $cronExpression = $this->getConfigSchedule($jobConfig) ?: null; ++ } ++ ++ if (!$cronExpression) { ++ if (isset($jobConfig['schedule'])) { ++ $cronExpression = $jobConfig['schedule']; ++ } ++ } ++ return $cronExpression; ++ } ++ ++ /** ++ * Clean up scheduled jobs that do not match their cron expression anymore ++ * This can happen when you change the cron expression and flush the cache ++ * ++ * @return $this ++ */ ++ private function cleanupScheduleMismatches() ++ { ++ /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ foreach ($this->invalid as $jobCode => $scheduledAtList) { ++ $scheduleResource->getConnection()->delete($scheduleResource->getMainTable(), [ ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code = ?' => $jobCode, ++ 'scheduled_at in (?)' => $scheduledAtList, ++ ]); ++ } ++ return $this; ++ } ++ ++ /** ++ * @return array ++ */ ++ private function getJobs() ++ { ++ if ($this->jobs === null) { ++ $this->jobs = $this->_config->getJobs(); ++ } ++ return $this->jobs; ++ } ++ ++ /** ++ * Get CronGroup Configuration Value ++ * ++ * @param $groupId ++ * @return int ++ */ ++ private function getCronGroupConfigurationValue($groupId, $path) ++ { ++ return $this->_scopeConfig->getValue( ++ 'system/cron/' . $groupId . '/' . $path, ++ \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ ); ++ return $scheduleLifetime; ++ } ++ ++ /** ++ * Is Group In Filter ++ * ++ * @param $groupId ++ * @return bool ++ */ ++ private function isGroupInFilter($groupId): bool ++ { ++ return !($this->_request->getParam('group') !== null ++ && trim($this->_request->getParam('group'), "'") !== $groupId); ++ } ++ ++ /** ++ * Process pending jobs ++ * ++ * @param $groupId ++ * @param $jobsRoot ++ * @param $currentTime ++ */ ++ private function processPendingJobs($groupId, $jobsRoot, $currentTime) ++ { ++ $procesedJobs = []; ++ $pendingJobs = $this->getPendingSchedules($groupId); ++ /** @var \Magento\Cron\Model\Schedule $schedule */ ++ foreach ($pendingJobs as $schedule) { ++ if (isset($procesedJobs[$schedule->getJobCode()])) { ++ // process only on job per run ++ continue; ++ } ++ $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; ++ if (!$jobConfig) { ++ continue; ++ } ++ ++ $scheduledTime = strtotime($schedule->getScheduledAt()); ++ if ($scheduledTime > $currentTime) { ++ continue; ++ } ++ ++ try { ++ if ($schedule->tryLockJob()) { ++ $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); ++ } ++ } catch (\Exception $e) { ++ $schedule->setMessages($e->getMessage()); ++ if ($schedule->getStatus() === Schedule::STATUS_ERROR) { ++ $this->logger->critical($e); ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_MISSED ++ && $this->state->getMode() === State::MODE_DEVELOPER ++ ) { ++ $this->logger->error( ++ sprintf( ++ "%s Schedule Id: %s Job Code: %s", ++ $schedule->getMessages(), ++ $schedule->getScheduleId(), ++ $schedule->getJobCode() ++ ) ++ ); ++ } ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_SUCCESS) { ++ $procesedJobs[$schedule->getJobCode()] = true; ++ } ++ $schedule->save(); ++ } ++ } + } + +diff -Naur a/vendor/magento/module-cron/Model/Schedule.php b/vendor/magento/module-cron/Model/Schedule.php +--- a/vendor/magento/module-cron/Model/Schedule.php ++++ b/vendor/magento/module-cron/Model/Schedule.php +@@ -1,18 +1,18 @@ + ++ * @api ++ * @since 100.0.2 + */ + class Schedule extends \Magento\Framework\Model\AbstractModel + { +@@ -45,20 +46,28 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + const STATUS_ERROR = 'error'; + + /** ++ * @var TimezoneInterface ++ */ ++ private $timezoneConverter; ++ ++ /** + * @param \Magento\Framework\Model\Context $context + * @param \Magento\Framework\Registry $registry + * @param \Magento\Framework\Model\ResourceModel\AbstractResource $resource + * @param \Magento\Framework\Data\Collection\AbstractDb $resourceCollection + * @param array $data ++ * @param TimezoneInterface $timezoneConverter + */ + public function __construct( + \Magento\Framework\Model\Context $context, + \Magento\Framework\Registry $registry, + \Magento\Framework\Model\ResourceModel\AbstractResource $resource = null, + \Magento\Framework\Data\Collection\AbstractDb $resourceCollection = null, +- array $data = [] ++ array $data = [], ++ TimezoneInterface $timezoneConverter = null + ) { + parent::__construct($context, $registry, $resource, $resourceCollection, $data); ++ $this->timezoneConverter = $timezoneConverter ?: ObjectManager::getInstance()->get(TimezoneInterface::class); + } + + /** +@@ -66,7 +75,7 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + */ + public function _construct() + { +- $this->_init('Magento\Cron\Model\ResourceModel\Schedule'); ++ $this->_init(\Magento\Cron\Model\ResourceModel\Schedule::class); + } + + /** +@@ -101,6 +110,9 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + return false; + } + if (!is_numeric($time)) { ++ //convert time from UTC to admin store timezone ++ //we assume that all schedules in configuration (crontab.xml and DB tables) are in admin store timezone ++ $time = $this->timezoneConverter->date($time)->format('Y-m-d H:i'); + $time = strtotime($time); + } + $match = $this->matchCronExpression($e[0], strftime('%M', $time)) +@@ -221,16 +233,17 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + } + + /** +- * Sets a job to STATUS_RUNNING only if it is currently in STATUS_PENDING. +- * Returns true if status was changed and false otherwise. ++ * Lock the cron job so no other scheduled instances run simultaneously. + * +- * This is used to implement locking for cron jobs. ++ * Sets a job to STATUS_RUNNING only if it is currently in STATUS_PENDING ++ * and no other jobs of the same code are currently in STATUS_RUNNING. ++ * Returns true if status was changed and false otherwise. + * + * @return boolean + */ + public function tryLockJob() + { +- if ($this->_getResource()->trySetJobStatusAtomic( ++ if ($this->_getResource()->trySetJobUniqueStatusAtomic( + $this->getId(), + self::STATUS_RUNNING, + self::STATUS_PENDING + +diff -Naur a/vendor/magento/module-cron/Model/ResourceModel/Schedule.php b/vendor/magento/module-cron/Model/ResourceModel/Schedule.php +index dca4e22..25dd02c 100644 +--- a/vendor/magento/module-cron/Model/ResourceModel/Schedule.php ++++ b/vendor/magento/module-cron/Model/ResourceModel/Schedule.php +@@ -8,7 +8,8 @@ namespace Magento\Cron\Model\ResourceModel; + /** + * Schedule resource + * +- * @author Magento Core Team ++ * @api ++ * @since 100.0.2 + */ + class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + { +@@ -23,9 +24,10 @@ class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + } + + /** +- * If job is currently in $currentStatus, set it to $newStatus +- * and return true. Otherwise, return false and do not change the job. +- * This method is used to implement locking for cron jobs. ++ * Sets new schedule status only if it's in the expected current status. ++ * ++ * If schedule is currently in $currentStatus, set it to $newStatus and ++ * return true. Otherwise, return false. + * + * @param string $scheduleId + * @param string $newStatus +@@ -45,4 +47,49 @@ class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + } + return false; + } ++ ++ /** ++ * Sets schedule status only if no existing schedules with the same job code ++ * have that status. This is used to implement locking for cron jobs. ++ * ++ * If the schedule is currently in $currentStatus and there are no existing ++ * schedules with the same job code and $newStatus, set the schedule to ++ * $newStatus and return true. Otherwise, return false. ++ * ++ * @param string $scheduleId ++ * @param string $newStatus ++ * @param string $currentStatus ++ * @return bool ++ * @since 100.2.0 ++ */ ++ public function trySetJobUniqueStatusAtomic($scheduleId, $newStatus, $currentStatus) ++ { ++ $connection = $this->getConnection(); ++ ++ // this condition added to avoid cron jobs locking after incorrect termination of running job ++ $match = $connection->quoteInto( ++ 'existing.job_code = current.job_code ' . ++ 'AND (existing.executed_at > UTC_TIMESTAMP() - INTERVAL 1 DAY OR existing.executed_at IS NULL) ' . ++ 'AND existing.status = ?', ++ $newStatus ++ ); ++ ++ $selectIfUnlocked = $connection->select() ++ ->joinLeft( ++ ['existing' => $this->getTable('cron_schedule')], ++ $match, ++ ['status' => new \Zend_Db_Expr($connection->quote($newStatus))] ++ ) ++ ->where('current.schedule_id = ?', $scheduleId) ++ ->where('current.status = ?', $currentStatus) ++ ->where('existing.schedule_id IS NULL'); ++ ++ $update = $connection->updateFromSelect($selectIfUnlocked, ['current' => $this->getTable('cron_schedule')]); ++ $result = $connection->query($update)->rowCount(); ++ ++ if ($result == 1) { ++ return true; ++ } ++ return false; ++ } + } diff --git a/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.4.patch b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.4.patch new file mode 100644 index 0000000..be3173c --- /dev/null +++ b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.4.patch @@ -0,0 +1,924 @@ +diff -Naur a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +--- a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php ++++ b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +@@ -1,6 +1,6 @@ + _objectManager = $objectManager; + $this->_scheduleFactory = $scheduleFactory; +@@ -134,8 +169,11 @@ class ProcessCronQueueObserver implements ObserverInterface + $this->_scopeConfig = $scopeConfig; + $this->_request = $request; + $this->_shell = $shell; +- $this->timezone = $timezone; ++ $this->dateTime = $dateTime; + $this->phpExecutableFinder = $phpExecutableFinderFactory->create(); ++ $this->logger = $logger; ++ $this->state = $state; ++ $this->statProfiler = $statFactory->create(); + } + + /** +@@ -151,26 +189,29 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + public function execute(\Magento\Framework\Event\Observer $observer) + { +- $pendingJobs = $this->_getPendingSchedules(); +- $currentTime = $this->timezone->scopeTimeStamp(); ++ ++ $currentTime = $this->dateTime->gmtTimestamp(); + $jobGroupsRoot = $this->_config->getJobs(); ++ // sort jobs groups to start from used in separated process ++ uksort( ++ $jobGroupsRoot, ++ function ($a, $b) { ++ return $this->getCronGroupConfigurationValue($b, 'use_separate_process') ++ - $this->getCronGroupConfigurationValue($a, 'use_separate_process'); ++ } ++ ); + + $phpPath = $this->phpExecutableFinder->find() ?: 'php'; + + foreach ($jobGroupsRoot as $groupId => $jobsRoot) { +- if ($this->_request->getParam('group') !== null +- && $this->_request->getParam('group') !== '\'' . ($groupId) . '\'' +- && $this->_request->getParam('group') !== $groupId) { ++ if (!$this->isGroupInFilter($groupId)) { + continue; + } +- if (($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1') && ( +- $this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/use_separate_process', +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ) == 1 +- )) { ++ if ($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1' ++ && $this->getCronGroupConfigurationValue($groupId, 'use_separate_process') == 1 ++ ) { + $this->_shell->execute( +- $phpPath . ' %s cron:run --group=' . $groupId . ' --' . CLI::INPUT_KEY_BOOTSTRAP . '=' ++ $phpPath . ' %s cron:run --group=' . $groupId . ' --' . Cli::INPUT_KEY_BOOTSTRAP . '=' + . self::STANDALONE_PROCESS_STARTED . '=1', + [ + BP . '/bin/magento' +@@ -179,29 +220,9 @@ class ProcessCronQueueObserver implements ObserverInterface + continue; + } + +- foreach ($pendingJobs as $schedule) { +- $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; +- if (!$jobConfig) { +- continue; +- } +- +- $scheduledTime = strtotime($schedule->getScheduledAt()); +- if ($scheduledTime > $currentTime) { +- continue; +- } +- +- try { +- if ($schedule->tryLockJob()) { +- $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); +- } +- } catch (\Exception $e) { +- $schedule->setMessages($e->getMessage()); +- } +- $schedule->save(); +- } +- +- $this->_generate($groupId); +- $this->_cleanup($groupId); ++ $this->cleanupJobs($groupId, $currentTime); ++ $this->generateSchedules($groupId); ++ $this->processPendingJobs($groupId, $jobsRoot, $currentTime); + } + } + +@@ -218,58 +239,105 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function _runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId) + { +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $jobCode = $schedule->getJobCode(); ++ $scheduleLifetime = $this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_LIFETIME); + $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + if ($scheduledTime < $currentTime - $scheduleLifetime) { + $schedule->setStatus(Schedule::STATUS_MISSED); ++ $this->logger->info(sprintf('Cron Job %s is missed', $jobCode)); + throw new \Exception('Too late for the schedule'); + } + + if (!isset($jobConfig['instance'], $jobConfig['method'])) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception('No callbacks found'); + } + $model = $this->_objectManager->create($jobConfig['instance']); + $callback = [$model, $jobConfig['method']]; + if (!is_callable($callback)) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception( + sprintf('Invalid callback: %s::%s can\'t be called', $jobConfig['instance'], $jobConfig['method']) + ); + } + +- $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->timezone->scopeTimeStamp()))->save(); ++ $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp()))->save(); + ++ $this->startProfiling(); + try { ++ $this->logger->info(sprintf('Cron Job %s is run', $jobCode)); + call_user_func_array($callback, [$schedule]); + } catch (\Exception $e) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf( ++ 'Cron Job %s has an error. Statistics: %s %s', ++ $jobCode, ++ $this->getProfilingStat(), $e->getMessage() ++ )); + throw $e; ++ } finally { ++ $this->stopProfiling(); + } + + $schedule->setStatus(Schedule::STATUS_SUCCESS)->setFinishedAt(strftime( + '%Y-%m-%d %H:%M:%S', +- $this->timezone->scopeTimeStamp() ++ $this->dateTime->gmtTimestamp() ++ )); ++ ++ $this->logger->info(sprintf( ++ 'Cron Job %s is successfully finished. Statistics: %s', ++ $jobCode, ++ $this->getProfilingStat() + )); + } + + /** ++ * Starts profiling ++ * ++ * @return void ++ */ ++ private function startProfiling() ++ { ++ $this->statProfiler->clear(); ++ $this->statProfiler->start('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Stops profiling ++ * ++ * @return void ++ */ ++ private function stopProfiling() ++ { ++ $this->statProfiler->stop('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Retrieves statistics in the JSON format ++ * ++ * @return string ++ */ ++ private function getProfilingStat() ++ { ++ $stat = $this->statProfiler->get('job'); ++ unset($stat[Stat::START]); ++ return json_encode($stat); ++ } ++ ++ /** + * Return job collection from data base with status 'pending' + * + * @return \Magento\Cron\Model\ResourceModel\Schedule\Collection + */ +- protected function _getPendingSchedules() ++ private function getPendingSchedules($groupId) + { +- if (!$this->_pendingSchedules) { +- $this->_pendingSchedules = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- Schedule::STATUS_PENDING +- )->load(); +- } +- return $this->_pendingSchedules; ++ $jobs = $this->getJobs(); ++ $pendingJobs = $this->_scheduleFactory->create()->getCollection(); ++ $pendingJobs->addFieldToFilter('status', Schedule::STATUS_PENDING); ++ $pendingJobs->addFieldToFilter('job_code', ['in' => array_keys($jobs[$groupId])]); ++ return $pendingJobs; + } + + /** +@@ -278,22 +346,32 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param string $groupId + * @return $this + */ +- protected function _generate($groupId) ++ private function generateSchedules($groupId) + { + /** + * check if schedule generation is needed + */ + $lastRun = (int)$this->_cache->load(self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId); +- $rawSchedulePeriod = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_GENERATE_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ $rawSchedulePeriod = (int)$this->getCronGroupConfigurationValue( ++ $groupId, ++ self::XML_PATH_SCHEDULE_GENERATE_EVERY + ); + $schedulePeriod = $rawSchedulePeriod * self::SECONDS_IN_MINUTE; +- if ($lastRun > $this->timezone->scopeTimeStamp() - $schedulePeriod) { ++ if ($lastRun > $this->dateTime->gmtTimestamp() - $schedulePeriod) { + return $this; + } + +- $schedules = $this->_getPendingSchedules(); ++ /** ++ * save time schedules generation was ran with no expiration ++ */ ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, ++ ['crontab'], ++ null ++ ); ++ ++ $schedules = $this->getPendingSchedules($groupId); + $exists = []; + /** @var Schedule $schedule */ + foreach ($schedules as $schedule) { +@@ -303,18 +381,10 @@ class ProcessCronQueueObserver implements ObserverInterface + /** + * generate global crontab jobs + */ +- $jobs = $this->_config->getJobs(); ++ $jobs = $this->getJobs(); ++ $this->invalid = []; + $this->_generateJobs($jobs[$groupId], $exists, $groupId); +- +- /** +- * save time schedules generation was ran with no expiration +- */ +- $this->_cache->save( +- $this->timezone->scopeTimeStamp(), +- self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, +- ['crontab'], +- null +- ); ++ $this->cleanupScheduleMismatches(); + + return $this; + } +@@ -325,22 +395,12 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param array $jobs + * @param array $exists + * @param string $groupId +- * @return $this ++ * @return void + */ + protected function _generateJobs($jobs, $exists, $groupId) + { + foreach ($jobs as $jobCode => $jobConfig) { +- $cronExpression = null; +- if (isset($jobConfig['config_path'])) { +- $cronExpression = $this->getConfigSchedule($jobConfig) ?: null; +- } +- +- if (!$cronExpression) { +- if (isset($jobConfig['schedule'])) { +- $cronExpression = $jobConfig['schedule']; +- } +- } +- ++ $cronExpression = $this->getCronExpression($jobConfig); + if (!$cronExpression) { + continue; + } +@@ -348,75 +408,60 @@ class ProcessCronQueueObserver implements ObserverInterface + $timeInterval = $this->getScheduleTimeInterval($groupId); + $this->saveSchedule($jobCode, $cronExpression, $timeInterval, $exists); + } +- return $this; + } + + /** +- * Clean existed jobs ++ * Clean expired jobs + * +- * @param string $groupId +- * @return $this ++ * @param $groupId ++ * @param $currentTime ++ * @return void + */ +- protected function _cleanup($groupId) ++ private function cleanupJobs($groupId, $currentTime) + { + // check if history cleanup is needed + $lastCleanup = (int)$this->_cache->load(self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId); +- $historyCleanUp = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_CLEANUP_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); +- if ($lastCleanup > $this->timezone->scopeTimeStamp() - $historyCleanUp * self::SECONDS_IN_MINUTE) { ++ $historyCleanUp = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_CLEANUP_EVERY); ++ if ($lastCleanup > $this->dateTime->gmtTimestamp() - $historyCleanUp * self::SECONDS_IN_MINUTE) { + return $this; + } +- +- // check how long the record should stay unprocessed before marked as MISSED +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ // save time history cleanup was ran with no expiration ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, ++ ['crontab'], ++ null + ); +- $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + +- /** +- * @var \Magento\Cron\Model\ResourceModel\Schedule\Collection $history +- */ +- $history = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- ['in' => [Schedule::STATUS_SUCCESS, Schedule::STATUS_MISSED, Schedule::STATUS_ERROR]] +- )->load(); ++ $this->cleanupDisabledJobs($groupId); + +- $historySuccess = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_SUCCESS, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); +- $historyFailure = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_FAILURE, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $historySuccess = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_SUCCESS); ++ $historyFailure = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_FAILURE); + $historyLifetimes = [ + Schedule::STATUS_SUCCESS => $historySuccess * self::SECONDS_IN_MINUTE, + Schedule::STATUS_MISSED => $historyFailure * self::SECONDS_IN_MINUTE, + Schedule::STATUS_ERROR => $historyFailure * self::SECONDS_IN_MINUTE, ++ Schedule::STATUS_PENDING => max($historyFailure, $historySuccess) * self::SECONDS_IN_MINUTE, + ]; + +- $now = $this->timezone->scopeTimeStamp(); +- /** @var Schedule $record */ +- foreach ($history as $record) { +- $checkTime = $record->getExecutedAt() ? strtotime($record->getExecutedAt()) : +- strtotime($record->getScheduledAt()) + $scheduleLifetime; +- if ($checkTime < $now - $historyLifetimes[$record->getStatus()]) { +- $record->delete(); +- } ++ $jobs = $this->getJobs()[$groupId]; ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $connection = $scheduleResource->getConnection(); ++ $count = 0; ++ foreach ($historyLifetimes as $status => $time) { ++ $count += $connection->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => $status, ++ 'job_code in (?)' => array_keys($jobs), ++ 'created_at < ?' => $connection->formatDate($currentTime - $time) ++ ] ++ ); + } + +- // save time history cleanup was ran with no expiration +- $this->_cache->save( +- $this->timezone->scopeTimeStamp(), +- self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, +- ['crontab'], +- null +- ); +- +- return $this; ++ if ($count) { ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } + } + + /** +@@ -442,19 +487,25 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function saveSchedule($jobCode, $cronExpression, $timeInterval, $exists) + { +- $currentTime = $this->timezone->scopeTimeStamp(); ++ $currentTime = $this->dateTime->gmtTimestamp(); + $timeAhead = $currentTime + $timeInterval; + for ($time = $currentTime; $time < $timeAhead; $time += self::SECONDS_IN_MINUTE) { +- $ts = strftime('%Y-%m-%d %H:%M:00', $time); +- if (!empty($exists[$jobCode . '/' . $ts])) { +- // already scheduled ++ $scheduledAt = strftime('%Y-%m-%d %H:%M:00', $time); ++ $alreadyScheduled = !empty($exists[$jobCode . '/' . $scheduledAt]); ++ $schedule = $this->createSchedule($jobCode, $cronExpression, $time); ++ $valid = $schedule->trySchedule(); ++ if (!$valid) { ++ if ($alreadyScheduled) { ++ if (!isset($this->invalid[$jobCode])) { ++ $this->invalid[$jobCode] = []; ++ } ++ $this->invalid[$jobCode][] = $scheduledAt; ++ } + continue; + } +- $schedule = $this->generateSchedule($jobCode, $cronExpression, $time); +- if ($schedule->trySchedule()) { ++ if (!$alreadyScheduled) { + // time matches cron expression + $schedule->save(); +- return; + } + } + } +@@ -465,13 +516,13 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param int $time + * @return Schedule + */ +- protected function generateSchedule($jobCode, $cronExpression, $time) ++ protected function createSchedule($jobCode, $cronExpression, $time) + { + $schedule = $this->_scheduleFactory->create() + ->setCronExpr($cronExpression) + ->setJobCode($jobCode) + ->setStatus(Schedule::STATUS_PENDING) +- ->setCreatedAt(strftime('%Y-%m-%d %H:%M:%S', $this->timezone->scopeTimeStamp())) ++ ->setCreatedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp())) + ->setScheduledAt(strftime('%Y-%m-%d %H:%M', $time)); + + return $schedule; +@@ -483,12 +534,174 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function getScheduleTimeInterval($groupId) + { +- $scheduleAheadFor = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_AHEAD_FOR, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $scheduleAheadFor = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_AHEAD_FOR); + $scheduleAheadFor = $scheduleAheadFor * self::SECONDS_IN_MINUTE; + + return $scheduleAheadFor; + } ++ ++ /** ++ * Clean up scheduled jobs that are disabled in the configuration ++ * This can happen when you turn off a cron job in the config and flush the cache ++ * ++ * @param string $groupId ++ * @return void ++ */ ++ private function cleanupDisabledJobs($groupId) ++ { ++ $jobs = $this->getJobs(); ++ $jobsToCleanup = []; ++ foreach ($jobs[$groupId] as $jobCode => $jobConfig) { ++ if (!$this->getCronExpression($jobConfig)) { ++ /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ ++ $jobsToCleanup[] = $jobCode; ++ } ++ } ++ ++ if (count($jobsToCleanup) > 0) { ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $count = $scheduleResource->getConnection()->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code in (?)' => $jobsToCleanup, ++ ] ++ ); ++ ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } ++ } ++ ++ /** ++ * @param array $jobConfig ++ * @return null|string ++ */ ++ private function getCronExpression($jobConfig) ++ { ++ $cronExpression = null; ++ if (isset($jobConfig['config_path'])) { ++ $cronExpression = $this->getConfigSchedule($jobConfig) ?: null; ++ } ++ ++ if (!$cronExpression) { ++ if (isset($jobConfig['schedule'])) { ++ $cronExpression = $jobConfig['schedule']; ++ } ++ } ++ return $cronExpression; ++ } ++ ++ /** ++ * Clean up scheduled jobs that do not match their cron expression anymore ++ * This can happen when you change the cron expression and flush the cache ++ * ++ * @return $this ++ */ ++ private function cleanupScheduleMismatches() ++ { ++ /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ foreach ($this->invalid as $jobCode => $scheduledAtList) { ++ $scheduleResource->getConnection()->delete($scheduleResource->getMainTable(), [ ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code = ?' => $jobCode, ++ 'scheduled_at in (?)' => $scheduledAtList, ++ ]); ++ } ++ return $this; ++ } ++ ++ /** ++ * @return array ++ */ ++ private function getJobs() ++ { ++ if ($this->jobs === null) { ++ $this->jobs = $this->_config->getJobs(); ++ } ++ return $this->jobs; ++ } ++ ++ /** ++ * Get CronGroup Configuration Value ++ * ++ * @param $groupId ++ * @return int ++ */ ++ private function getCronGroupConfigurationValue($groupId, $path) ++ { ++ return $this->_scopeConfig->getValue( ++ 'system/cron/' . $groupId . '/' . $path, ++ \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ ); ++ return $scheduleLifetime; ++ } ++ ++ /** ++ * Is Group In Filter ++ * ++ * @param $groupId ++ * @return bool ++ */ ++ private function isGroupInFilter($groupId): bool ++ { ++ return !($this->_request->getParam('group') !== null ++ && trim($this->_request->getParam('group'), "'") !== $groupId); ++ } ++ ++ /** ++ * Process pending jobs ++ * ++ * @param $groupId ++ * @param $jobsRoot ++ * @param $currentTime ++ */ ++ private function processPendingJobs($groupId, $jobsRoot, $currentTime) ++ { ++ $procesedJobs = []; ++ $pendingJobs = $this->getPendingSchedules($groupId); ++ /** @var \Magento\Cron\Model\Schedule $schedule */ ++ foreach ($pendingJobs as $schedule) { ++ if (isset($procesedJobs[$schedule->getJobCode()])) { ++ // process only on job per run ++ continue; ++ } ++ $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; ++ if (!$jobConfig) { ++ continue; ++ } ++ ++ $scheduledTime = strtotime($schedule->getScheduledAt()); ++ if ($scheduledTime > $currentTime) { ++ continue; ++ } ++ ++ try { ++ if ($schedule->tryLockJob()) { ++ $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); ++ } ++ } catch (\Exception $e) { ++ $schedule->setMessages($e->getMessage()); ++ if ($schedule->getStatus() === Schedule::STATUS_ERROR) { ++ $this->logger->critical($e); ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_MISSED ++ && $this->state->getMode() === State::MODE_DEVELOPER ++ ) { ++ $this->logger->error( ++ sprintf( ++ "%s Schedule Id: %s Job Code: %s", ++ $schedule->getMessages(), ++ $schedule->getScheduleId(), ++ $schedule->getJobCode() ++ ) ++ ); ++ } ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_SUCCESS) { ++ $procesedJobs[$schedule->getJobCode()] = true; ++ } ++ $schedule->save(); ++ } ++ } + } + +diff -Naur a/vendor/magento/module-cron/Model/ResourceModel/Schedule.php b/vendor/magento/module-cron/Model/ResourceModel/Schedule.php +index dc401e3..25dd02c 100644 +--- a/vendor/magento/module-cron/Model/ResourceModel/Schedule.php ++++ b/vendor/magento/module-cron/Model/ResourceModel/Schedule.php +@@ -1,6 +1,6 @@ + ++ * @api ++ * @since 100.0.2 + */ + class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + { +@@ -23,9 +24,10 @@ class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + } + + /** +- * If job is currently in $currentStatus, set it to $newStatus +- * and return true. Otherwise, return false and do not change the job. +- * This method is used to implement locking for cron jobs. ++ * Sets new schedule status only if it's in the expected current status. ++ * ++ * If schedule is currently in $currentStatus, set it to $newStatus and ++ * return true. Otherwise, return false. + * + * @param string $scheduleId + * @param string $newStatus +@@ -45,4 +47,49 @@ class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + } + return false; + } ++ ++ /** ++ * Sets schedule status only if no existing schedules with the same job code ++ * have that status. This is used to implement locking for cron jobs. ++ * ++ * If the schedule is currently in $currentStatus and there are no existing ++ * schedules with the same job code and $newStatus, set the schedule to ++ * $newStatus and return true. Otherwise, return false. ++ * ++ * @param string $scheduleId ++ * @param string $newStatus ++ * @param string $currentStatus ++ * @return bool ++ * @since 100.2.0 ++ */ ++ public function trySetJobUniqueStatusAtomic($scheduleId, $newStatus, $currentStatus) ++ { ++ $connection = $this->getConnection(); ++ ++ // this condition added to avoid cron jobs locking after incorrect termination of running job ++ $match = $connection->quoteInto( ++ 'existing.job_code = current.job_code ' . ++ 'AND (existing.executed_at > UTC_TIMESTAMP() - INTERVAL 1 DAY OR existing.executed_at IS NULL) ' . ++ 'AND existing.status = ?', ++ $newStatus ++ ); ++ ++ $selectIfUnlocked = $connection->select() ++ ->joinLeft( ++ ['existing' => $this->getTable('cron_schedule')], ++ $match, ++ ['status' => new \Zend_Db_Expr($connection->quote($newStatus))] ++ ) ++ ->where('current.schedule_id = ?', $scheduleId) ++ ->where('current.status = ?', $currentStatus) ++ ->where('existing.schedule_id IS NULL'); ++ ++ $update = $connection->updateFromSelect($selectIfUnlocked, ['current' => $this->getTable('cron_schedule')]); ++ $result = $connection->query($update)->rowCount(); ++ ++ if ($result == 1) { ++ return true; ++ } ++ return false; ++ } + } + +diff -Naur a/vendor/magento/module-cron/Model/Schedule.php b/vendor/magento/module-cron/Model/Schedule.php +--- a/vendor/magento/module-cron/Model/Schedule.php ++++ b/vendor/magento/module-cron/Model/Schedule.php +@@ -1,18 +1,18 @@ + ++ * @api ++ * @since 100.0.2 + */ + class Schedule extends \Magento\Framework\Model\AbstractModel + { +@@ -45,20 +46,28 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + const STATUS_ERROR = 'error'; + + /** ++ * @var TimezoneInterface ++ */ ++ private $timezoneConverter; ++ ++ /** + * @param \Magento\Framework\Model\Context $context + * @param \Magento\Framework\Registry $registry + * @param \Magento\Framework\Model\ResourceModel\AbstractResource $resource + * @param \Magento\Framework\Data\Collection\AbstractDb $resourceCollection + * @param array $data ++ * @param TimezoneInterface $timezoneConverter + */ + public function __construct( + \Magento\Framework\Model\Context $context, + \Magento\Framework\Registry $registry, + \Magento\Framework\Model\ResourceModel\AbstractResource $resource = null, + \Magento\Framework\Data\Collection\AbstractDb $resourceCollection = null, +- array $data = [] ++ array $data = [], ++ TimezoneInterface $timezoneConverter = null + ) { + parent::__construct($context, $registry, $resource, $resourceCollection, $data); ++ $this->timezoneConverter = $timezoneConverter ?: ObjectManager::getInstance()->get(TimezoneInterface::class); + } + + /** +@@ -66,7 +75,7 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + */ + public function _construct() + { +- $this->_init('Magento\Cron\Model\ResourceModel\Schedule'); ++ $this->_init(\Magento\Cron\Model\ResourceModel\Schedule::class); + } + + /** +@@ -101,6 +110,9 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + return false; + } + if (!is_numeric($time)) { ++ //convert time from UTC to admin store timezone ++ //we assume that all schedules in configuration (crontab.xml and DB tables) are in admin store timezone ++ $time = $this->timezoneConverter->date($time)->format('Y-m-d H:i'); + $time = strtotime($time); + } + $match = $this->matchCronExpression($e[0], strftime('%M', $time)) +@@ -221,16 +233,17 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + } + + /** +- * Sets a job to STATUS_RUNNING only if it is currently in STATUS_PENDING. +- * Returns true if status was changed and false otherwise. ++ * Lock the cron job so no other scheduled instances run simultaneously. + * +- * This is used to implement locking for cron jobs. ++ * Sets a job to STATUS_RUNNING only if it is currently in STATUS_PENDING ++ * and no other jobs of the same code are currently in STATUS_RUNNING. ++ * Returns true if status was changed and false otherwise. + * + * @return boolean + */ + public function tryLockJob() + { +- if ($this->_getResource()->trySetJobStatusAtomic( ++ if ($this->_getResource()->trySetJobUniqueStatusAtomic( + $this->getId(), + self::STATUS_RUNNING, + self::STATUS_PENDING diff --git a/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.5.patch b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.5.patch new file mode 100644 index 0000000..363704e --- /dev/null +++ b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.1.5.patch @@ -0,0 +1,924 @@ +diff -Naur a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +--- a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php ++++ b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +@@ -1,6 +1,6 @@ + _objectManager = $objectManager; + $this->_scheduleFactory = $scheduleFactory; +@@ -134,8 +169,11 @@ class ProcessCronQueueObserver implements ObserverInterface + $this->_scopeConfig = $scopeConfig; + $this->_request = $request; + $this->_shell = $shell; +- $this->timezone = $timezone; ++ $this->dateTime = $dateTime; + $this->phpExecutableFinder = $phpExecutableFinderFactory->create(); ++ $this->logger = $logger; ++ $this->state = $state; ++ $this->statProfiler = $statFactory->create(); + } + + /** +@@ -151,26 +189,29 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + public function execute(\Magento\Framework\Event\Observer $observer) + { +- $pendingJobs = $this->_getPendingSchedules(); +- $currentTime = $this->timezone->scopeTimeStamp(); ++ ++ $currentTime = $this->dateTime->gmtTimestamp(); + $jobGroupsRoot = $this->_config->getJobs(); ++ // sort jobs groups to start from used in separated process ++ uksort( ++ $jobGroupsRoot, ++ function ($a, $b) { ++ return $this->getCronGroupConfigurationValue($b, 'use_separate_process') ++ - $this->getCronGroupConfigurationValue($a, 'use_separate_process'); ++ } ++ ); + + $phpPath = $this->phpExecutableFinder->find() ?: 'php'; + + foreach ($jobGroupsRoot as $groupId => $jobsRoot) { +- if ($this->_request->getParam('group') !== null +- && $this->_request->getParam('group') !== '\'' . ($groupId) . '\'' +- && $this->_request->getParam('group') !== $groupId) { ++ if (!$this->isGroupInFilter($groupId)) { + continue; + } +- if (($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1') && ( +- $this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/use_separate_process', +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ) == 1 +- )) { ++ if ($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1' ++ && $this->getCronGroupConfigurationValue($groupId, 'use_separate_process') == 1 ++ ) { + $this->_shell->execute( +- $phpPath . ' %s cron:run --group=' . $groupId . ' --' . CLI::INPUT_KEY_BOOTSTRAP . '=' ++ $phpPath . ' %s cron:run --group=' . $groupId . ' --' . Cli::INPUT_KEY_BOOTSTRAP . '=' + . self::STANDALONE_PROCESS_STARTED . '=1', + [ + BP . '/bin/magento' +@@ -179,29 +220,9 @@ class ProcessCronQueueObserver implements ObserverInterface + continue; + } + +- foreach ($pendingJobs as $schedule) { +- $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; +- if (!$jobConfig) { +- continue; +- } +- +- $scheduledTime = strtotime($schedule->getScheduledAt()); +- if ($scheduledTime > $currentTime) { +- continue; +- } +- +- try { +- if ($schedule->tryLockJob()) { +- $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); +- } +- } catch (\Exception $e) { +- $schedule->setMessages($e->getMessage()); +- } +- $schedule->save(); +- } +- +- $this->_generate($groupId); +- $this->_cleanup($groupId); ++ $this->cleanupJobs($groupId, $currentTime); ++ $this->generateSchedules($groupId); ++ $this->processPendingJobs($groupId, $jobsRoot, $currentTime); + } + } + +@@ -218,58 +239,105 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function _runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId) + { +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $jobCode = $schedule->getJobCode(); ++ $scheduleLifetime = $this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_LIFETIME); + $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + if ($scheduledTime < $currentTime - $scheduleLifetime) { + $schedule->setStatus(Schedule::STATUS_MISSED); ++ $this->logger->info(sprintf('Cron Job %s is missed', $jobCode)); + throw new \Exception('Too late for the schedule'); + } + + if (!isset($jobConfig['instance'], $jobConfig['method'])) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception('No callbacks found'); + } + $model = $this->_objectManager->create($jobConfig['instance']); + $callback = [$model, $jobConfig['method']]; + if (!is_callable($callback)) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception( + sprintf('Invalid callback: %s::%s can\'t be called', $jobConfig['instance'], $jobConfig['method']) + ); + } + +- $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->timezone->scopeTimeStamp()))->save(); ++ $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp()))->save(); + ++ $this->startProfiling(); + try { ++ $this->logger->info(sprintf('Cron Job %s is run', $jobCode)); + call_user_func_array($callback, [$schedule]); + } catch (\Exception $e) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf( ++ 'Cron Job %s has an error. Statistics: %s %s', ++ $jobCode, ++ $this->getProfilingStat(), $e->getMessage() ++ )); + throw $e; ++ } finally { ++ $this->stopProfiling(); + } + + $schedule->setStatus(Schedule::STATUS_SUCCESS)->setFinishedAt(strftime( + '%Y-%m-%d %H:%M:%S', +- $this->timezone->scopeTimeStamp() ++ $this->dateTime->gmtTimestamp() ++ )); ++ ++ $this->logger->info(sprintf( ++ 'Cron Job %s is successfully finished. Statistics: %s', ++ $jobCode, ++ $this->getProfilingStat() + )); + } + + /** ++ * Starts profiling ++ * ++ * @return void ++ */ ++ private function startProfiling() ++ { ++ $this->statProfiler->clear(); ++ $this->statProfiler->start('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Stops profiling ++ * ++ * @return void ++ */ ++ private function stopProfiling() ++ { ++ $this->statProfiler->stop('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Retrieves statistics in the JSON format ++ * ++ * @return string ++ */ ++ private function getProfilingStat() ++ { ++ $stat = $this->statProfiler->get('job'); ++ unset($stat[Stat::START]); ++ return json_encode($stat); ++ } ++ ++ /** + * Return job collection from data base with status 'pending' + * + * @return \Magento\Cron\Model\ResourceModel\Schedule\Collection + */ +- protected function _getPendingSchedules() ++ private function getPendingSchedules($groupId) + { +- if (!$this->_pendingSchedules) { +- $this->_pendingSchedules = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- Schedule::STATUS_PENDING +- )->load(); +- } +- return $this->_pendingSchedules; ++ $jobs = $this->getJobs(); ++ $pendingJobs = $this->_scheduleFactory->create()->getCollection(); ++ $pendingJobs->addFieldToFilter('status', Schedule::STATUS_PENDING); ++ $pendingJobs->addFieldToFilter('job_code', ['in' => array_keys($jobs[$groupId])]); ++ return $pendingJobs; + } + + /** +@@ -278,22 +346,32 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param string $groupId + * @return $this + */ +- protected function _generate($groupId) ++ private function generateSchedules($groupId) + { + /** + * check if schedule generation is needed + */ + $lastRun = (int)$this->_cache->load(self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId); +- $rawSchedulePeriod = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_GENERATE_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ $rawSchedulePeriod = (int)$this->getCronGroupConfigurationValue( ++ $groupId, ++ self::XML_PATH_SCHEDULE_GENERATE_EVERY + ); + $schedulePeriod = $rawSchedulePeriod * self::SECONDS_IN_MINUTE; +- if ($lastRun > $this->timezone->scopeTimeStamp() - $schedulePeriod) { ++ if ($lastRun > $this->dateTime->gmtTimestamp() - $schedulePeriod) { + return $this; + } + +- $schedules = $this->_getPendingSchedules(); ++ /** ++ * save time schedules generation was ran with no expiration ++ */ ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, ++ ['crontab'], ++ null ++ ); ++ ++ $schedules = $this->getPendingSchedules($groupId); + $exists = []; + /** @var Schedule $schedule */ + foreach ($schedules as $schedule) { +@@ -303,18 +381,10 @@ class ProcessCronQueueObserver implements ObserverInterface + /** + * generate global crontab jobs + */ +- $jobs = $this->_config->getJobs(); ++ $jobs = $this->getJobs(); ++ $this->invalid = []; + $this->_generateJobs($jobs[$groupId], $exists, $groupId); +- +- /** +- * save time schedules generation was ran with no expiration +- */ +- $this->_cache->save( +- $this->timezone->scopeTimeStamp(), +- self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, +- ['crontab'], +- null +- ); ++ $this->cleanupScheduleMismatches(); + + return $this; + } +@@ -325,22 +395,12 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param array $jobs + * @param array $exists + * @param string $groupId +- * @return $this ++ * @return void + */ + protected function _generateJobs($jobs, $exists, $groupId) + { + foreach ($jobs as $jobCode => $jobConfig) { +- $cronExpression = null; +- if (isset($jobConfig['config_path'])) { +- $cronExpression = $this->getConfigSchedule($jobConfig) ?: null; +- } +- +- if (!$cronExpression) { +- if (isset($jobConfig['schedule'])) { +- $cronExpression = $jobConfig['schedule']; +- } +- } +- ++ $cronExpression = $this->getCronExpression($jobConfig); + if (!$cronExpression) { + continue; + } +@@ -348,75 +408,60 @@ class ProcessCronQueueObserver implements ObserverInterface + $timeInterval = $this->getScheduleTimeInterval($groupId); + $this->saveSchedule($jobCode, $cronExpression, $timeInterval, $exists); + } +- return $this; + } + + /** +- * Clean existed jobs ++ * Clean expired jobs + * +- * @param string $groupId +- * @return $this ++ * @param $groupId ++ * @param $currentTime ++ * @return void + */ +- protected function _cleanup($groupId) ++ private function cleanupJobs($groupId, $currentTime) + { + // check if history cleanup is needed + $lastCleanup = (int)$this->_cache->load(self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId); +- $historyCleanUp = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_CLEANUP_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); +- if ($lastCleanup > $this->timezone->scopeTimeStamp() - $historyCleanUp * self::SECONDS_IN_MINUTE) { ++ $historyCleanUp = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_CLEANUP_EVERY); ++ if ($lastCleanup > $this->dateTime->gmtTimestamp() - $historyCleanUp * self::SECONDS_IN_MINUTE) { + return $this; + } +- +- // check how long the record should stay unprocessed before marked as MISSED +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ // save time history cleanup was ran with no expiration ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, ++ ['crontab'], ++ null + ); +- $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + +- /** +- * @var \Magento\Cron\Model\ResourceModel\Schedule\Collection $history +- */ +- $history = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- ['in' => [Schedule::STATUS_SUCCESS, Schedule::STATUS_MISSED, Schedule::STATUS_ERROR]] +- )->load(); ++ $this->cleanupDisabledJobs($groupId); + +- $historySuccess = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_SUCCESS, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); +- $historyFailure = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_FAILURE, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $historySuccess = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_SUCCESS); ++ $historyFailure = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_FAILURE); + $historyLifetimes = [ + Schedule::STATUS_SUCCESS => $historySuccess * self::SECONDS_IN_MINUTE, + Schedule::STATUS_MISSED => $historyFailure * self::SECONDS_IN_MINUTE, + Schedule::STATUS_ERROR => $historyFailure * self::SECONDS_IN_MINUTE, ++ Schedule::STATUS_PENDING => max($historyFailure, $historySuccess) * self::SECONDS_IN_MINUTE, + ]; + +- $now = $this->timezone->scopeTimeStamp(); +- /** @var Schedule $record */ +- foreach ($history as $record) { +- $checkTime = $record->getExecutedAt() ? strtotime($record->getExecutedAt()) : +- strtotime($record->getScheduledAt()) + $scheduleLifetime; +- if ($checkTime < $now - $historyLifetimes[$record->getStatus()]) { +- $record->delete(); +- } ++ $jobs = $this->getJobs()[$groupId]; ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $connection = $scheduleResource->getConnection(); ++ $count = 0; ++ foreach ($historyLifetimes as $status => $time) { ++ $count += $connection->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => $status, ++ 'job_code in (?)' => array_keys($jobs), ++ 'created_at < ?' => $connection->formatDate($currentTime - $time) ++ ] ++ ); + } + +- // save time history cleanup was ran with no expiration +- $this->_cache->save( +- $this->timezone->scopeTimeStamp(), +- self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, +- ['crontab'], +- null +- ); +- +- return $this; ++ if ($count) { ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } + } + + /** +@@ -442,19 +487,25 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function saveSchedule($jobCode, $cronExpression, $timeInterval, $exists) + { +- $currentTime = $this->timezone->scopeTimeStamp(); ++ $currentTime = $this->dateTime->gmtTimestamp(); + $timeAhead = $currentTime + $timeInterval; + for ($time = $currentTime; $time < $timeAhead; $time += self::SECONDS_IN_MINUTE) { +- $ts = strftime('%Y-%m-%d %H:%M:00', $time); +- if (!empty($exists[$jobCode . '/' . $ts])) { +- // already scheduled ++ $scheduledAt = strftime('%Y-%m-%d %H:%M:00', $time); ++ $alreadyScheduled = !empty($exists[$jobCode . '/' . $scheduledAt]); ++ $schedule = $this->createSchedule($jobCode, $cronExpression, $time); ++ $valid = $schedule->trySchedule(); ++ if (!$valid) { ++ if ($alreadyScheduled) { ++ if (!isset($this->invalid[$jobCode])) { ++ $this->invalid[$jobCode] = []; ++ } ++ $this->invalid[$jobCode][] = $scheduledAt; ++ } + continue; + } +- $schedule = $this->generateSchedule($jobCode, $cronExpression, $time); +- if ($schedule->trySchedule()) { ++ if (!$alreadyScheduled) { + // time matches cron expression + $schedule->save(); +- return; + } + } + } +@@ -465,13 +516,13 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param int $time + * @return Schedule + */ +- protected function generateSchedule($jobCode, $cronExpression, $time) ++ protected function createSchedule($jobCode, $cronExpression, $time) + { + $schedule = $this->_scheduleFactory->create() + ->setCronExpr($cronExpression) + ->setJobCode($jobCode) + ->setStatus(Schedule::STATUS_PENDING) +- ->setCreatedAt(strftime('%Y-%m-%d %H:%M:%S', $this->timezone->scopeTimeStamp())) ++ ->setCreatedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp())) + ->setScheduledAt(strftime('%Y-%m-%d %H:%M', $time)); + + return $schedule; +@@ -483,12 +534,174 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function getScheduleTimeInterval($groupId) + { +- $scheduleAheadFor = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_AHEAD_FOR, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $scheduleAheadFor = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_AHEAD_FOR); + $scheduleAheadFor = $scheduleAheadFor * self::SECONDS_IN_MINUTE; + + return $scheduleAheadFor; + } ++ ++ /** ++ * Clean up scheduled jobs that are disabled in the configuration ++ * This can happen when you turn off a cron job in the config and flush the cache ++ * ++ * @param string $groupId ++ * @return void ++ */ ++ private function cleanupDisabledJobs($groupId) ++ { ++ $jobs = $this->getJobs(); ++ $jobsToCleanup = []; ++ foreach ($jobs[$groupId] as $jobCode => $jobConfig) { ++ if (!$this->getCronExpression($jobConfig)) { ++ /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ ++ $jobsToCleanup[] = $jobCode; ++ } ++ } ++ ++ if (count($jobsToCleanup) > 0) { ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $count = $scheduleResource->getConnection()->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code in (?)' => $jobsToCleanup, ++ ] ++ ); ++ ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } ++ } ++ ++ /** ++ * @param array $jobConfig ++ * @return null|string ++ */ ++ private function getCronExpression($jobConfig) ++ { ++ $cronExpression = null; ++ if (isset($jobConfig['config_path'])) { ++ $cronExpression = $this->getConfigSchedule($jobConfig) ?: null; ++ } ++ ++ if (!$cronExpression) { ++ if (isset($jobConfig['schedule'])) { ++ $cronExpression = $jobConfig['schedule']; ++ } ++ } ++ return $cronExpression; ++ } ++ ++ /** ++ * Clean up scheduled jobs that do not match their cron expression anymore ++ * This can happen when you change the cron expression and flush the cache ++ * ++ * @return $this ++ */ ++ private function cleanupScheduleMismatches() ++ { ++ /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ foreach ($this->invalid as $jobCode => $scheduledAtList) { ++ $scheduleResource->getConnection()->delete($scheduleResource->getMainTable(), [ ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code = ?' => $jobCode, ++ 'scheduled_at in (?)' => $scheduledAtList, ++ ]); ++ } ++ return $this; ++ } ++ ++ /** ++ * @return array ++ */ ++ private function getJobs() ++ { ++ if ($this->jobs === null) { ++ $this->jobs = $this->_config->getJobs(); ++ } ++ return $this->jobs; ++ } ++ ++ /** ++ * Get CronGroup Configuration Value ++ * ++ * @param $groupId ++ * @return int ++ */ ++ private function getCronGroupConfigurationValue($groupId, $path) ++ { ++ return $this->_scopeConfig->getValue( ++ 'system/cron/' . $groupId . '/' . $path, ++ \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ ); ++ return $scheduleLifetime; ++ } ++ ++ /** ++ * Is Group In Filter ++ * ++ * @param $groupId ++ * @return bool ++ */ ++ private function isGroupInFilter($groupId): bool ++ { ++ return !($this->_request->getParam('group') !== null ++ && trim($this->_request->getParam('group'), "'") !== $groupId); ++ } ++ ++ /** ++ * Process pending jobs ++ * ++ * @param $groupId ++ * @param $jobsRoot ++ * @param $currentTime ++ */ ++ private function processPendingJobs($groupId, $jobsRoot, $currentTime) ++ { ++ $procesedJobs = []; ++ $pendingJobs = $this->getPendingSchedules($groupId); ++ /** @var \Magento\Cron\Model\Schedule $schedule */ ++ foreach ($pendingJobs as $schedule) { ++ if (isset($procesedJobs[$schedule->getJobCode()])) { ++ // process only on job per run ++ continue; ++ } ++ $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; ++ if (!$jobConfig) { ++ continue; ++ } ++ ++ $scheduledTime = strtotime($schedule->getScheduledAt()); ++ if ($scheduledTime > $currentTime) { ++ continue; ++ } ++ ++ try { ++ if ($schedule->tryLockJob()) { ++ $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); ++ } ++ } catch (\Exception $e) { ++ $schedule->setMessages($e->getMessage()); ++ if ($schedule->getStatus() === Schedule::STATUS_ERROR) { ++ $this->logger->critical($e); ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_MISSED ++ && $this->state->getMode() === State::MODE_DEVELOPER ++ ) { ++ $this->logger->error( ++ sprintf( ++ "%s Schedule Id: %s Job Code: %s", ++ $schedule->getMessages(), ++ $schedule->getScheduleId(), ++ $schedule->getJobCode() ++ ) ++ ); ++ } ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_SUCCESS) { ++ $procesedJobs[$schedule->getJobCode()] = true; ++ } ++ $schedule->save(); ++ } ++ } + } + +diff -Naur a/vendor/magento/module-cron/Model/Schedule.php b/vendor/magento/module-cron/Model/Schedule.php +--- a/vendor/magento/module-cron/Model/Schedule.php ++++ b/vendor/magento/module-cron/Model/Schedule.php +@@ -1,18 +1,18 @@ + ++ * @api ++ * @since 100.0.2 + */ + class Schedule extends \Magento\Framework\Model\AbstractModel + { +@@ -45,20 +46,28 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + const STATUS_ERROR = 'error'; + + /** ++ * @var TimezoneInterface ++ */ ++ private $timezoneConverter; ++ ++ /** + * @param \Magento\Framework\Model\Context $context + * @param \Magento\Framework\Registry $registry + * @param \Magento\Framework\Model\ResourceModel\AbstractResource $resource + * @param \Magento\Framework\Data\Collection\AbstractDb $resourceCollection + * @param array $data ++ * @param TimezoneInterface $timezoneConverter + */ + public function __construct( + \Magento\Framework\Model\Context $context, + \Magento\Framework\Registry $registry, + \Magento\Framework\Model\ResourceModel\AbstractResource $resource = null, + \Magento\Framework\Data\Collection\AbstractDb $resourceCollection = null, +- array $data = [] ++ array $data = [], ++ TimezoneInterface $timezoneConverter = null + ) { + parent::__construct($context, $registry, $resource, $resourceCollection, $data); ++ $this->timezoneConverter = $timezoneConverter ?: ObjectManager::getInstance()->get(TimezoneInterface::class); + } + + /** +@@ -66,7 +75,7 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + */ + public function _construct() + { +- $this->_init('Magento\Cron\Model\ResourceModel\Schedule'); ++ $this->_init(\Magento\Cron\Model\ResourceModel\Schedule::class); + } + + /** +@@ -101,6 +110,9 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + return false; + } + if (!is_numeric($time)) { ++ //convert time from UTC to admin store timezone ++ //we assume that all schedules in configuration (crontab.xml and DB tables) are in admin store timezone ++ $time = $this->timezoneConverter->date($time)->format('Y-m-d H:i'); + $time = strtotime($time); + } + $match = $this->matchCronExpression($e[0], strftime('%M', $time)) +@@ -221,16 +233,17 @@ class Schedule extends \Magento\Framework\Model\AbstractModel + } + + /** +- * Sets a job to STATUS_RUNNING only if it is currently in STATUS_PENDING. +- * Returns true if status was changed and false otherwise. ++ * Lock the cron job so no other scheduled instances run simultaneously. + * +- * This is used to implement locking for cron jobs. ++ * Sets a job to STATUS_RUNNING only if it is currently in STATUS_PENDING ++ * and no other jobs of the same code are currently in STATUS_RUNNING. ++ * Returns true if status was changed and false otherwise. + * + * @return boolean + */ + public function tryLockJob() + { +- if ($this->_getResource()->trySetJobStatusAtomic( ++ if ($this->_getResource()->trySetJobUniqueStatusAtomic( + $this->getId(), + self::STATUS_RUNNING, + self::STATUS_PENDING + +diff -Naur a/vendor/magento/module-cron/Model/ResourceModel/Schedule.php b/vendor/magento/module-cron/Model/ResourceModel/Schedule.php +index dca4e22..25dd02c 100644 +--- a/vendor/magento/module-cron/Model/ResourceModel/Schedule.php ++++ b/vendor/magento/module-cron/Model/ResourceModel/Schedule.php +@@ -1,6 +1,6 @@ + ++ * @api ++ * @since 100.0.2 + */ + class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + { +@@ -23,9 +24,10 @@ class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + } + + /** +- * If job is currently in $currentStatus, set it to $newStatus +- * and return true. Otherwise, return false and do not change the job. +- * This method is used to implement locking for cron jobs. ++ * Sets new schedule status only if it's in the expected current status. ++ * ++ * If schedule is currently in $currentStatus, set it to $newStatus and ++ * return true. Otherwise, return false. + * + * @param string $scheduleId + * @param string $newStatus +@@ -45,4 +47,49 @@ class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + } + return false; + } ++ ++ /** ++ * Sets schedule status only if no existing schedules with the same job code ++ * have that status. This is used to implement locking for cron jobs. ++ * ++ * If the schedule is currently in $currentStatus and there are no existing ++ * schedules with the same job code and $newStatus, set the schedule to ++ * $newStatus and return true. Otherwise, return false. ++ * ++ * @param string $scheduleId ++ * @param string $newStatus ++ * @param string $currentStatus ++ * @return bool ++ * @since 100.2.0 ++ */ ++ public function trySetJobUniqueStatusAtomic($scheduleId, $newStatus, $currentStatus) ++ { ++ $connection = $this->getConnection(); ++ ++ // this condition added to avoid cron jobs locking after incorrect termination of running job ++ $match = $connection->quoteInto( ++ 'existing.job_code = current.job_code ' . ++ 'AND (existing.executed_at > UTC_TIMESTAMP() - INTERVAL 1 DAY OR existing.executed_at IS NULL) ' . ++ 'AND existing.status = ?', ++ $newStatus ++ ); ++ ++ $selectIfUnlocked = $connection->select() ++ ->joinLeft( ++ ['existing' => $this->getTable('cron_schedule')], ++ $match, ++ ['status' => new \Zend_Db_Expr($connection->quote($newStatus))] ++ ) ++ ->where('current.schedule_id = ?', $scheduleId) ++ ->where('current.status = ?', $currentStatus) ++ ->where('existing.schedule_id IS NULL'); ++ ++ $update = $connection->updateFromSelect($selectIfUnlocked, ['current' => $this->getTable('cron_schedule')]); ++ $result = $connection->query($update)->rowCount(); ++ ++ if ($result == 1) { ++ return true; ++ } ++ return false; ++ } + } diff --git a/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.0.patch b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.0.patch new file mode 100644 index 0000000..17cdb82 --- /dev/null +++ b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.0.patch @@ -0,0 +1,616 @@ +diff -Naur a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +index f772a6c..d760e92 100644 +--- a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php ++++ b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +@@ -13,6 +13,8 @@ use Magento\Framework\App\State; + use Magento\Framework\Console\Cli; + use Magento\Framework\Event\ObserverInterface; + use \Magento\Cron\Model\Schedule; ++use Magento\Framework\Profiler\Driver\Standard\Stat; ++use Magento\Framework\Profiler\Driver\Standard\StatFactory; + + /** + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) +@@ -127,6 +129,11 @@ class ProcessCronQueueObserver implements ObserverInterface + private $jobs; + + /** ++ * @var Stat ++ */ ++ private $statProfiler; ++ ++ /** + * @param \Magento\Framework\ObjectManagerInterface $objectManager + * @param \Magento\Cron\Model\ScheduleFactory $scheduleFactory + * @param \Magento\Framework\App\CacheInterface $cache +@@ -138,6 +145,7 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory + * @param \Psr\Log\LoggerInterface $logger + * @param \Magento\Framework\App\State $state ++ * @param StatFactory $statFactory + * @SuppressWarnings(PHPMD.ExcessiveParameterList) + */ + public function __construct( +@@ -151,7 +159,8 @@ class ProcessCronQueueObserver implements ObserverInterface + \Magento\Framework\Stdlib\DateTime\DateTime $dateTime, + \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory, + \Psr\Log\LoggerInterface $logger, +- \Magento\Framework\App\State $state ++ \Magento\Framework\App\State $state, ++ StatFactory $statFactory + ) { + $this->_objectManager = $objectManager; + $this->_scheduleFactory = $scheduleFactory; +@@ -164,6 +173,7 @@ class ProcessCronQueueObserver implements ObserverInterface + $this->phpExecutableFinder = $phpExecutableFinderFactory->create(); + $this->logger = $logger; + $this->state = $state; ++ $this->statProfiler = $statFactory->create(); + } + + /** +@@ -179,27 +189,26 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + public function execute(\Magento\Framework\Event\Observer $observer) + { +- $pendingJobs = $this->_getPendingSchedules(); ++ + $currentTime = $this->dateTime->gmtTimestamp(); + $jobGroupsRoot = $this->_config->getJobs(); ++ // sort jobs groups to start from used in separated process ++ uksort( ++ $jobGroupsRoot, ++ function ($a, $b) { ++ return $this->getCronGroupConfigurationValue($b, 'use_separate_process') ++ - $this->getCronGroupConfigurationValue($a, 'use_separate_process'); ++ } ++ ); + + $phpPath = $this->phpExecutableFinder->find() ?: 'php'; + + foreach ($jobGroupsRoot as $groupId => $jobsRoot) { +- $this->_cleanup($groupId); +- $this->_generate($groupId); +- if ($this->_request->getParam('group') !== null +- && $this->_request->getParam('group') !== '\'' . ($groupId) . '\'' +- && $this->_request->getParam('group') !== $groupId +- ) { ++ if (!$this->isGroupInFilter($groupId)) { + continue; + } +- if (($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1') && ( +- $this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/use_separate_process', +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ) == 1 +- ) ++ if ($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1' ++ && $this->getCronGroupConfigurationValue($groupId, 'use_separate_process') == 1 + ) { + $this->_shell->execute( + $phpPath . ' %s cron:run --group=' . $groupId . ' --' . Cli::INPUT_KEY_BOOTSTRAP . '=' +@@ -211,42 +220,9 @@ class ProcessCronQueueObserver implements ObserverInterface + continue; + } + +- /** @var \Magento\Cron\Model\Schedule $schedule */ +- foreach ($pendingJobs as $schedule) { +- $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; +- if (!$jobConfig) { +- continue; +- } +- +- $scheduledTime = strtotime($schedule->getScheduledAt()); +- if ($scheduledTime > $currentTime) { +- continue; +- } +- +- try { +- if ($schedule->tryLockJob()) { +- $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); +- } +- } catch (\Exception $e) { +- $schedule->setMessages($e->getMessage()); +- if ($schedule->getStatus() === Schedule::STATUS_ERROR) { +- $this->logger->critical($e); +- } +- if ($schedule->getStatus() === Schedule::STATUS_MISSED +- && $this->state->getMode() === State::MODE_DEVELOPER +- ) { +- $this->logger->info( +- sprintf( +- "%s Schedule Id: %s Job Code: %s", +- $schedule->getMessages(), +- $schedule->getScheduleId(), +- $schedule->getJobCode() +- ) +- ); +- } +- } +- $schedule->save(); +- } ++ $this->cleanupJobs($groupId, $currentTime); ++ $this->generateSchedules($groupId); ++ $this->processPendingJobs($groupId, $jobsRoot, $currentTime); + } + } + +@@ -263,24 +239,25 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function _runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId) + { +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $jobCode = $schedule->getJobCode(); ++ $scheduleLifetime = $this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_LIFETIME); + $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + if ($scheduledTime < $currentTime - $scheduleLifetime) { + $schedule->setStatus(Schedule::STATUS_MISSED); ++ $this->logger->info(sprintf('Cron Job %s is missed', $jobCode)); + throw new \Exception('Too late for the schedule'); + } + + if (!isset($jobConfig['instance'], $jobConfig['method'])) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception('No callbacks found'); + } + $model = $this->_objectManager->create($jobConfig['instance']); + $callback = [$model, $jobConfig['method']]; + if (!is_callable($callback)) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception( + sprintf('Invalid callback: %s::%s can\'t be called', $jobConfig['instance'], $jobConfig['method']) + ); +@@ -288,17 +265,65 @@ class ProcessCronQueueObserver implements ObserverInterface + + $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp()))->save(); + ++ $this->startProfiling(); + try { ++ $this->logger->info(sprintf('Cron Job %s is run', $jobCode)); + call_user_func_array($callback, [$schedule]); + } catch (\Exception $e) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf( ++ 'Cron Job %s has an error. Statistics: %s %s', ++ $jobCode, ++ $this->getProfilingStat(), $e->getMessage() ++ )); + throw $e; ++ } finally { ++ $this->stopProfiling(); + } + + $schedule->setStatus(Schedule::STATUS_SUCCESS)->setFinishedAt(strftime( + '%Y-%m-%d %H:%M:%S', + $this->dateTime->gmtTimestamp() + )); ++ ++ $this->logger->info(sprintf( ++ 'Cron Job %s is successfully finished. Statistics: %s', ++ $jobCode, ++ $this->getProfilingStat() ++ )); ++ } ++ ++ /** ++ * Starts profiling ++ * ++ * @return void ++ */ ++ private function startProfiling() ++ { ++ $this->statProfiler->clear(); ++ $this->statProfiler->start('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Stops profiling ++ * ++ * @return void ++ */ ++ private function stopProfiling() ++ { ++ $this->statProfiler->stop('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Retrieves statistics in the JSON format ++ * ++ * @return string ++ */ ++ private function getProfilingStat() ++ { ++ $stat = $this->statProfiler->get('job'); ++ unset($stat[Stat::START]); ++ return json_encode($stat); + } + + /** +@@ -306,15 +331,13 @@ class ProcessCronQueueObserver implements ObserverInterface + * + * @return \Magento\Cron\Model\ResourceModel\Schedule\Collection + */ +- protected function _getPendingSchedules() ++ private function getPendingSchedules($groupId) + { +- if (!$this->_pendingSchedules) { +- $this->_pendingSchedules = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- Schedule::STATUS_PENDING +- )->load(); +- } +- return $this->_pendingSchedules; ++ $jobs = $this->getJobs(); ++ $pendingJobs = $this->_scheduleFactory->create()->getCollection(); ++ $pendingJobs->addFieldToFilter('status', Schedule::STATUS_PENDING); ++ $pendingJobs->addFieldToFilter('job_code', ['in' => array_keys($jobs[$groupId])]); ++ return $pendingJobs; + } + + /** +@@ -323,22 +346,32 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param string $groupId + * @return $this + */ +- protected function _generate($groupId) ++ private function generateSchedules($groupId) + { + /** + * check if schedule generation is needed + */ + $lastRun = (int)$this->_cache->load(self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId); +- $rawSchedulePeriod = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_GENERATE_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ $rawSchedulePeriod = (int)$this->getCronGroupConfigurationValue( ++ $groupId, ++ self::XML_PATH_SCHEDULE_GENERATE_EVERY + ); + $schedulePeriod = $rawSchedulePeriod * self::SECONDS_IN_MINUTE; + if ($lastRun > $this->dateTime->gmtTimestamp() - $schedulePeriod) { + return $this; + } + +- $schedules = $this->_getPendingSchedules(); ++ /** ++ * save time schedules generation was ran with no expiration ++ */ ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, ++ ['crontab'], ++ null ++ ); ++ ++ $schedules = $this->getPendingSchedules($groupId); + $exists = []; + /** @var Schedule $schedule */ + foreach ($schedules as $schedule) { +@@ -353,16 +386,6 @@ class ProcessCronQueueObserver implements ObserverInterface + $this->_generateJobs($jobs[$groupId], $exists, $groupId); + $this->cleanupScheduleMismatches(); + +- /** +- * save time schedules generation was ran with no expiration +- */ +- $this->_cache->save( +- $this->dateTime->gmtTimestamp(), +- self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, +- ['crontab'], +- null +- ); +- + return $this; + } + +@@ -372,7 +395,7 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param array $jobs + * @param array $exists + * @param string $groupId +- * @return $this ++ * @return void + */ + protected function _generateJobs($jobs, $exists, $groupId) + { +@@ -385,77 +408,60 @@ class ProcessCronQueueObserver implements ObserverInterface + $timeInterval = $this->getScheduleTimeInterval($groupId); + $this->saveSchedule($jobCode, $cronExpression, $timeInterval, $exists); + } +- return $this; + } + + /** + * Clean expired jobs + * +- * @param string $groupId +- * @return $this ++ * @param $groupId ++ * @param $currentTime ++ * @return void + */ +- protected function _cleanup($groupId) ++ private function cleanupJobs($groupId, $currentTime) + { +- $this->cleanupDisabledJobs($groupId); +- + // check if history cleanup is needed + $lastCleanup = (int)$this->_cache->load(self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId); +- $historyCleanUp = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_CLEANUP_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $historyCleanUp = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_CLEANUP_EVERY); + if ($lastCleanup > $this->dateTime->gmtTimestamp() - $historyCleanUp * self::SECONDS_IN_MINUTE) { + return $this; + } +- +- // check how long the record should stay unprocessed before marked as MISSED +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ // save time history cleanup was ran with no expiration ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, ++ ['crontab'], ++ null + ); +- $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + +- /** +- * @var \Magento\Cron\Model\ResourceModel\Schedule\Collection $history +- */ +- $history = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- ['in' => [Schedule::STATUS_SUCCESS, Schedule::STATUS_MISSED, Schedule::STATUS_ERROR]] +- )->load(); ++ $this->cleanupDisabledJobs($groupId); + +- $historySuccess = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_SUCCESS, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); +- $historyFailure = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_FAILURE, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $historySuccess = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_SUCCESS); ++ $historyFailure = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_FAILURE); + $historyLifetimes = [ + Schedule::STATUS_SUCCESS => $historySuccess * self::SECONDS_IN_MINUTE, + Schedule::STATUS_MISSED => $historyFailure * self::SECONDS_IN_MINUTE, + Schedule::STATUS_ERROR => $historyFailure * self::SECONDS_IN_MINUTE, ++ Schedule::STATUS_PENDING => max($historyFailure, $historySuccess) * self::SECONDS_IN_MINUTE, + ]; + +- $now = $this->dateTime->gmtTimestamp(); +- /** @var Schedule $record */ +- foreach ($history as $record) { +- $checkTime = $record->getExecutedAt() ? strtotime($record->getExecutedAt()) : +- strtotime($record->getScheduledAt()) + $scheduleLifetime; +- if ($checkTime < $now - $historyLifetimes[$record->getStatus()]) { +- $record->delete(); +- } ++ $jobs = $this->getJobs()[$groupId]; ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $connection = $scheduleResource->getConnection(); ++ $count = 0; ++ foreach ($historyLifetimes as $status => $time) { ++ $count += $connection->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => $status, ++ 'job_code in (?)' => array_keys($jobs), ++ 'created_at < ?' => $connection->formatDate($currentTime - $time) ++ ] ++ ); + } + +- // save time history cleanup was ran with no expiration +- $this->_cache->save( +- $this->dateTime->gmtTimestamp(), +- self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, +- ['crontab'], +- null +- ); +- +- return $this; ++ if ($count) { ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } + } + + /** +@@ -486,7 +492,7 @@ class ProcessCronQueueObserver implements ObserverInterface + for ($time = $currentTime; $time < $timeAhead; $time += self::SECONDS_IN_MINUTE) { + $scheduledAt = strftime('%Y-%m-%d %H:%M:00', $time); + $alreadyScheduled = !empty($exists[$jobCode . '/' . $scheduledAt]); +- $schedule = $this->generateSchedule($jobCode, $cronExpression, $time); ++ $schedule = $this->createSchedule($jobCode, $cronExpression, $time); + $valid = $schedule->trySchedule(); + if (!$valid) { + if ($alreadyScheduled) { +@@ -510,7 +516,7 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param int $time + * @return Schedule + */ +- protected function generateSchedule($jobCode, $cronExpression, $time) ++ protected function createSchedule($jobCode, $cronExpression, $time) + { + $schedule = $this->_scheduleFactory->create() + ->setCronExpr($cronExpression) +@@ -528,10 +534,7 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function getScheduleTimeInterval($groupId) + { +- $scheduleAheadFor = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_AHEAD_FOR, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $scheduleAheadFor = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_AHEAD_FOR); + $scheduleAheadFor = $scheduleAheadFor * self::SECONDS_IN_MINUTE; + + return $scheduleAheadFor; +@@ -547,16 +550,26 @@ class ProcessCronQueueObserver implements ObserverInterface + private function cleanupDisabledJobs($groupId) + { + $jobs = $this->getJobs(); ++ $jobsToCleanup = []; + foreach ($jobs[$groupId] as $jobCode => $jobConfig) { + if (!$this->getCronExpression($jobConfig)) { + /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ +- $scheduleResource = $this->_scheduleFactory->create()->getResource(); +- $scheduleResource->getConnection()->delete($scheduleResource->getMainTable(), [ +- 'status=?' => Schedule::STATUS_PENDING, +- 'job_code=?' => $jobCode, +- ]); ++ $jobsToCleanup[] = $jobCode; + } + } ++ ++ if (count($jobsToCleanup) > 0) { ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $count = $scheduleResource->getConnection()->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code in (?)' => $jobsToCleanup, ++ ] ++ ); ++ ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } + } + + /** +@@ -586,12 +599,12 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + private function cleanupScheduleMismatches() + { ++ /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); + foreach ($this->invalid as $jobCode => $scheduledAtList) { +- /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ +- $scheduleResource = $this->_scheduleFactory->create()->getResource(); + $scheduleResource->getConnection()->delete($scheduleResource->getMainTable(), [ +- 'status=?' => Schedule::STATUS_PENDING, +- 'job_code=?' => $jobCode, ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code = ?' => $jobCode, + 'scheduled_at in (?)' => $scheduledAtList, + ]); + } +@@ -608,4 +621,87 @@ class ProcessCronQueueObserver implements ObserverInterface + } + return $this->jobs; + } ++ ++ /** ++ * Get CronGroup Configuration Value ++ * ++ * @param $groupId ++ * @return int ++ */ ++ private function getCronGroupConfigurationValue($groupId, $path) ++ { ++ return $this->_scopeConfig->getValue( ++ 'system/cron/' . $groupId . '/' . $path, ++ \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ ); ++ return $scheduleLifetime; ++ } ++ ++ /** ++ * Is Group In Filter ++ * ++ * @param $groupId ++ * @return bool ++ */ ++ private function isGroupInFilter($groupId): bool ++ { ++ return !($this->_request->getParam('group') !== null ++ && trim($this->_request->getParam('group'), "'") !== $groupId); ++ } ++ ++ /** ++ * Process pending jobs ++ * ++ * @param $groupId ++ * @param $jobsRoot ++ * @param $currentTime ++ */ ++ private function processPendingJobs($groupId, $jobsRoot, $currentTime) ++ { ++ $procesedJobs = []; ++ $pendingJobs = $this->getPendingSchedules($groupId); ++ /** @var \Magento\Cron\Model\Schedule $schedule */ ++ foreach ($pendingJobs as $schedule) { ++ if (isset($procesedJobs[$schedule->getJobCode()])) { ++ // process only on job per run ++ continue; ++ } ++ $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; ++ if (!$jobConfig) { ++ continue; ++ } ++ ++ $scheduledTime = strtotime($schedule->getScheduledAt()); ++ if ($scheduledTime > $currentTime) { ++ continue; ++ } ++ ++ try { ++ if ($schedule->tryLockJob()) { ++ $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); ++ } ++ } catch (\Exception $e) { ++ $schedule->setMessages($e->getMessage()); ++ if ($schedule->getStatus() === Schedule::STATUS_ERROR) { ++ $this->logger->critical($e); ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_MISSED ++ && $this->state->getMode() === State::MODE_DEVELOPER ++ ) { ++ $this->logger->info( ++ sprintf( ++ "%s Schedule Id: %s Job Code: %s", ++ $schedule->getMessages(), ++ $schedule->getScheduleId(), ++ $schedule->getJobCode() ++ ) ++ ); ++ } ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_SUCCESS) { ++ $procesedJobs[$schedule->getJobCode()] = true; ++ } ++ $schedule->save(); ++ } ++ } + } + +diff -Naur a/vendor/magento/module-cron/Model/ResourceModel/Schedule.php b/vendor/magento/module-cron/Model/ResourceModel/Schedule.php +index a47227b..25dd02c 100644 +--- a/vendor/magento/module-cron/Model/ResourceModel/Schedule.php ++++ b/vendor/magento/module-cron/Model/ResourceModel/Schedule.php +@@ -66,7 +66,14 @@ class Schedule extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb + { + $connection = $this->getConnection(); + +- $match = $connection->quoteInto('existing.job_code = current.job_code AND existing.status = ?', $newStatus); ++ // this condition added to avoid cron jobs locking after incorrect termination of running job ++ $match = $connection->quoteInto( ++ 'existing.job_code = current.job_code ' . ++ 'AND (existing.executed_at > UTC_TIMESTAMP() - INTERVAL 1 DAY OR existing.executed_at IS NULL) ' . ++ 'AND existing.status = ?', ++ $newStatus ++ ); ++ + $selectIfUnlocked = $connection->select() + ->joinLeft( + ['existing' => $this->getTable('cron_schedule')], diff --git a/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.2.patch b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.2.patch new file mode 100644 index 0000000..94b780b --- /dev/null +++ b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.2.patch @@ -0,0 +1,595 @@ +diff -Naur a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +index f772a6c..d760e92 100644 +--- a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php ++++ b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +@@ -13,6 +13,8 @@ use Magento\Framework\App\State; + use Magento\Framework\Console\Cli; + use Magento\Framework\Event\ObserverInterface; + use \Magento\Cron\Model\Schedule; ++use Magento\Framework\Profiler\Driver\Standard\Stat; ++use Magento\Framework\Profiler\Driver\Standard\StatFactory; + + /** + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) +@@ -127,6 +129,11 @@ class ProcessCronQueueObserver implements ObserverInterface + private $jobs; + + /** ++ * @var Stat ++ */ ++ private $statProfiler; ++ ++ /** + * @param \Magento\Framework\ObjectManagerInterface $objectManager + * @param \Magento\Cron\Model\ScheduleFactory $scheduleFactory + * @param \Magento\Framework\App\CacheInterface $cache +@@ -138,6 +145,7 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory + * @param \Psr\Log\LoggerInterface $logger + * @param \Magento\Framework\App\State $state ++ * @param StatFactory $statFactory + * @SuppressWarnings(PHPMD.ExcessiveParameterList) + */ + public function __construct( +@@ -151,7 +159,8 @@ class ProcessCronQueueObserver implements ObserverInterface + \Magento\Framework\Stdlib\DateTime\DateTime $dateTime, + \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory, + \Psr\Log\LoggerInterface $logger, +- \Magento\Framework\App\State $state ++ \Magento\Framework\App\State $state, ++ StatFactory $statFactory + ) { + $this->_objectManager = $objectManager; + $this->_scheduleFactory = $scheduleFactory; +@@ -164,6 +173,7 @@ class ProcessCronQueueObserver implements ObserverInterface + $this->phpExecutableFinder = $phpExecutableFinderFactory->create(); + $this->logger = $logger; + $this->state = $state; ++ $this->statProfiler = $statFactory->create(); + } + + /** +@@ -179,27 +189,26 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + public function execute(\Magento\Framework\Event\Observer $observer) + { +- $pendingJobs = $this->_getPendingSchedules(); ++ + $currentTime = $this->dateTime->gmtTimestamp(); + $jobGroupsRoot = $this->_config->getJobs(); ++ // sort jobs groups to start from used in separated process ++ uksort( ++ $jobGroupsRoot, ++ function ($a, $b) { ++ return $this->getCronGroupConfigurationValue($b, 'use_separate_process') ++ - $this->getCronGroupConfigurationValue($a, 'use_separate_process'); ++ } ++ ); + + $phpPath = $this->phpExecutableFinder->find() ?: 'php'; + + foreach ($jobGroupsRoot as $groupId => $jobsRoot) { +- $this->_cleanup($groupId); +- $this->_generate($groupId); +- if ($this->_request->getParam('group') !== null +- && $this->_request->getParam('group') !== '\'' . ($groupId) . '\'' +- && $this->_request->getParam('group') !== $groupId +- ) { ++ if (!$this->isGroupInFilter($groupId)) { + continue; + } +- if (($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1') && ( +- $this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/use_separate_process', +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ) == 1 +- ) ++ if ($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1' ++ && $this->getCronGroupConfigurationValue($groupId, 'use_separate_process') == 1 + ) { + $this->_shell->execute( + $phpPath . ' %s cron:run --group=' . $groupId . ' --' . Cli::INPUT_KEY_BOOTSTRAP . '=' +@@ -211,42 +220,9 @@ class ProcessCronQueueObserver implements ObserverInterface + continue; + } + +- /** @var \Magento\Cron\Model\Schedule $schedule */ +- foreach ($pendingJobs as $schedule) { +- $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; +- if (!$jobConfig) { +- continue; +- } +- +- $scheduledTime = strtotime($schedule->getScheduledAt()); +- if ($scheduledTime > $currentTime) { +- continue; +- } +- +- try { +- if ($schedule->tryLockJob()) { +- $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); +- } +- } catch (\Exception $e) { +- $schedule->setMessages($e->getMessage()); +- if ($schedule->getStatus() === Schedule::STATUS_ERROR) { +- $this->logger->critical($e); +- } +- if ($schedule->getStatus() === Schedule::STATUS_MISSED +- && $this->state->getMode() === State::MODE_DEVELOPER +- ) { +- $this->logger->info( +- sprintf( +- "%s Schedule Id: %s Job Code: %s", +- $schedule->getMessages(), +- $schedule->getScheduleId(), +- $schedule->getJobCode() +- ) +- ); +- } +- } +- $schedule->save(); +- } ++ $this->cleanupJobs($groupId, $currentTime); ++ $this->generateSchedules($groupId); ++ $this->processPendingJobs($groupId, $jobsRoot, $currentTime); + } + } + +@@ -263,24 +239,25 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function _runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId) + { +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $jobCode = $schedule->getJobCode(); ++ $scheduleLifetime = $this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_LIFETIME); + $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + if ($scheduledTime < $currentTime - $scheduleLifetime) { + $schedule->setStatus(Schedule::STATUS_MISSED); ++ $this->logger->info(sprintf('Cron Job %s is missed', $jobCode)); + throw new \Exception('Too late for the schedule'); + } + + if (!isset($jobConfig['instance'], $jobConfig['method'])) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception('No callbacks found'); + } + $model = $this->_objectManager->create($jobConfig['instance']); + $callback = [$model, $jobConfig['method']]; + if (!is_callable($callback)) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception( + sprintf('Invalid callback: %s::%s can\'t be called', $jobConfig['instance'], $jobConfig['method']) + ); +@@ -288,17 +265,65 @@ class ProcessCronQueueObserver implements ObserverInterface + + $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp()))->save(); + ++ $this->startProfiling(); + try { ++ $this->logger->info(sprintf('Cron Job %s is run', $jobCode)); + call_user_func_array($callback, [$schedule]); + } catch (\Exception $e) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf( ++ 'Cron Job %s has an error. Statistics: %s %s', ++ $jobCode, ++ $this->getProfilingStat(), $e->getMessage() ++ )); + throw $e; ++ } finally { ++ $this->stopProfiling(); + } + + $schedule->setStatus(Schedule::STATUS_SUCCESS)->setFinishedAt(strftime( + '%Y-%m-%d %H:%M:%S', + $this->dateTime->gmtTimestamp() + )); ++ ++ $this->logger->info(sprintf( ++ 'Cron Job %s is successfully finished. Statistics: %s', ++ $jobCode, ++ $this->getProfilingStat() ++ )); ++ } ++ ++ /** ++ * Starts profiling ++ * ++ * @return void ++ */ ++ private function startProfiling() ++ { ++ $this->statProfiler->clear(); ++ $this->statProfiler->start('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Stops profiling ++ * ++ * @return void ++ */ ++ private function stopProfiling() ++ { ++ $this->statProfiler->stop('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Retrieves statistics in the JSON format ++ * ++ * @return string ++ */ ++ private function getProfilingStat() ++ { ++ $stat = $this->statProfiler->get('job'); ++ unset($stat[Stat::START]); ++ return json_encode($stat); + } + + /** +@@ -306,15 +331,13 @@ class ProcessCronQueueObserver implements ObserverInterface + * + * @return \Magento\Cron\Model\ResourceModel\Schedule\Collection + */ +- protected function _getPendingSchedules() ++ private function getPendingSchedules($groupId) + { +- if (!$this->_pendingSchedules) { +- $this->_pendingSchedules = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- Schedule::STATUS_PENDING +- )->load(); +- } +- return $this->_pendingSchedules; ++ $jobs = $this->getJobs(); ++ $pendingJobs = $this->_scheduleFactory->create()->getCollection(); ++ $pendingJobs->addFieldToFilter('status', Schedule::STATUS_PENDING); ++ $pendingJobs->addFieldToFilter('job_code', ['in' => array_keys($jobs[$groupId])]); ++ return $pendingJobs; + } + + /** +@@ -323,22 +346,32 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param string $groupId + * @return $this + */ +- protected function _generate($groupId) ++ private function generateSchedules($groupId) + { + /** + * check if schedule generation is needed + */ + $lastRun = (int)$this->_cache->load(self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId); +- $rawSchedulePeriod = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_GENERATE_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ $rawSchedulePeriod = (int)$this->getCronGroupConfigurationValue( ++ $groupId, ++ self::XML_PATH_SCHEDULE_GENERATE_EVERY + ); + $schedulePeriod = $rawSchedulePeriod * self::SECONDS_IN_MINUTE; + if ($lastRun > $this->dateTime->gmtTimestamp() - $schedulePeriod) { + return $this; + } + +- $schedules = $this->_getPendingSchedules(); ++ /** ++ * save time schedules generation was ran with no expiration ++ */ ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, ++ ['crontab'], ++ null ++ ); ++ ++ $schedules = $this->getPendingSchedules($groupId); + $exists = []; + /** @var Schedule $schedule */ + foreach ($schedules as $schedule) { +@@ -353,16 +386,6 @@ class ProcessCronQueueObserver implements ObserverInterface + $this->_generateJobs($jobs[$groupId], $exists, $groupId); + $this->cleanupScheduleMismatches(); + +- /** +- * save time schedules generation was ran with no expiration +- */ +- $this->_cache->save( +- $this->dateTime->gmtTimestamp(), +- self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, +- ['crontab'], +- null +- ); +- + return $this; + } + +@@ -372,7 +395,7 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param array $jobs + * @param array $exists + * @param string $groupId +- * @return $this ++ * @return void + */ + protected function _generateJobs($jobs, $exists, $groupId) + { +@@ -385,77 +408,60 @@ class ProcessCronQueueObserver implements ObserverInterface + $timeInterval = $this->getScheduleTimeInterval($groupId); + $this->saveSchedule($jobCode, $cronExpression, $timeInterval, $exists); + } +- return $this; + } + + /** + * Clean expired jobs + * +- * @param string $groupId +- * @return $this ++ * @param $groupId ++ * @param $currentTime ++ * @return void + */ +- protected function _cleanup($groupId) ++ private function cleanupJobs($groupId, $currentTime) + { +- $this->cleanupDisabledJobs($groupId); +- + // check if history cleanup is needed + $lastCleanup = (int)$this->_cache->load(self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId); +- $historyCleanUp = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_CLEANUP_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $historyCleanUp = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_CLEANUP_EVERY); + if ($lastCleanup > $this->dateTime->gmtTimestamp() - $historyCleanUp * self::SECONDS_IN_MINUTE) { + return $this; + } +- +- // check how long the record should stay unprocessed before marked as MISSED +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ // save time history cleanup was ran with no expiration ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, ++ ['crontab'], ++ null + ); +- $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + +- /** +- * @var \Magento\Cron\Model\ResourceModel\Schedule\Collection $history +- */ +- $history = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- ['in' => [Schedule::STATUS_SUCCESS, Schedule::STATUS_MISSED, Schedule::STATUS_ERROR]] +- )->load(); ++ $this->cleanupDisabledJobs($groupId); + +- $historySuccess = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_SUCCESS, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); +- $historyFailure = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_FAILURE, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $historySuccess = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_SUCCESS); ++ $historyFailure = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_FAILURE); + $historyLifetimes = [ + Schedule::STATUS_SUCCESS => $historySuccess * self::SECONDS_IN_MINUTE, + Schedule::STATUS_MISSED => $historyFailure * self::SECONDS_IN_MINUTE, + Schedule::STATUS_ERROR => $historyFailure * self::SECONDS_IN_MINUTE, ++ Schedule::STATUS_PENDING => max($historyFailure, $historySuccess) * self::SECONDS_IN_MINUTE, + ]; + +- $now = $this->dateTime->gmtTimestamp(); +- /** @var Schedule $record */ +- foreach ($history as $record) { +- $checkTime = $record->getExecutedAt() ? strtotime($record->getExecutedAt()) : +- strtotime($record->getScheduledAt()) + $scheduleLifetime; +- if ($checkTime < $now - $historyLifetimes[$record->getStatus()]) { +- $record->delete(); +- } ++ $jobs = $this->getJobs()[$groupId]; ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $connection = $scheduleResource->getConnection(); ++ $count = 0; ++ foreach ($historyLifetimes as $status => $time) { ++ $count += $connection->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => $status, ++ 'job_code in (?)' => array_keys($jobs), ++ 'created_at < ?' => $connection->formatDate($currentTime - $time) ++ ] ++ ); + } + +- // save time history cleanup was ran with no expiration +- $this->_cache->save( +- $this->dateTime->gmtTimestamp(), +- self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, +- ['crontab'], +- null +- ); +- +- return $this; ++ if ($count) { ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } + } + + /** +@@ -486,7 +492,7 @@ class ProcessCronQueueObserver implements ObserverInterface + for ($time = $currentTime; $time < $timeAhead; $time += self::SECONDS_IN_MINUTE) { + $scheduledAt = strftime('%Y-%m-%d %H:%M:00', $time); + $alreadyScheduled = !empty($exists[$jobCode . '/' . $scheduledAt]); +- $schedule = $this->generateSchedule($jobCode, $cronExpression, $time); ++ $schedule = $this->createSchedule($jobCode, $cronExpression, $time); + $valid = $schedule->trySchedule(); + if (!$valid) { + if ($alreadyScheduled) { +@@ -510,7 +516,7 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param int $time + * @return Schedule + */ +- protected function generateSchedule($jobCode, $cronExpression, $time) ++ protected function createSchedule($jobCode, $cronExpression, $time) + { + $schedule = $this->_scheduleFactory->create() + ->setCronExpr($cronExpression) +@@ -528,10 +534,7 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function getScheduleTimeInterval($groupId) + { +- $scheduleAheadFor = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_AHEAD_FOR, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $scheduleAheadFor = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_AHEAD_FOR); + $scheduleAheadFor = $scheduleAheadFor * self::SECONDS_IN_MINUTE; + + return $scheduleAheadFor; +@@ -547,16 +550,26 @@ class ProcessCronQueueObserver implements ObserverInterface + private function cleanupDisabledJobs($groupId) + { + $jobs = $this->getJobs(); ++ $jobsToCleanup = []; + foreach ($jobs[$groupId] as $jobCode => $jobConfig) { + if (!$this->getCronExpression($jobConfig)) { + /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ +- $scheduleResource = $this->_scheduleFactory->create()->getResource(); +- $scheduleResource->getConnection()->delete($scheduleResource->getMainTable(), [ +- 'status=?' => Schedule::STATUS_PENDING, +- 'job_code=?' => $jobCode, +- ]); ++ $jobsToCleanup[] = $jobCode; + } + } ++ ++ if (count($jobsToCleanup) > 0) { ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $count = $scheduleResource->getConnection()->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code in (?)' => $jobsToCleanup, ++ ] ++ ); ++ ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } + } + + /** +@@ -586,12 +599,12 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + private function cleanupScheduleMismatches() + { ++ /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); + foreach ($this->invalid as $jobCode => $scheduledAtList) { +- /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ +- $scheduleResource = $this->_scheduleFactory->create()->getResource(); + $scheduleResource->getConnection()->delete($scheduleResource->getMainTable(), [ +- 'status=?' => Schedule::STATUS_PENDING, +- 'job_code=?' => $jobCode, ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code = ?' => $jobCode, + 'scheduled_at in (?)' => $scheduledAtList, + ]); + } +@@ -608,4 +621,87 @@ class ProcessCronQueueObserver implements ObserverInterface + } + return $this->jobs; + } ++ ++ /** ++ * Get CronGroup Configuration Value ++ * ++ * @param $groupId ++ * @return int ++ */ ++ private function getCronGroupConfigurationValue($groupId, $path) ++ { ++ return $this->_scopeConfig->getValue( ++ 'system/cron/' . $groupId . '/' . $path, ++ \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ ); ++ return $scheduleLifetime; ++ } ++ ++ /** ++ * Is Group In Filter ++ * ++ * @param $groupId ++ * @return bool ++ */ ++ private function isGroupInFilter($groupId): bool ++ { ++ return !($this->_request->getParam('group') !== null ++ && trim($this->_request->getParam('group'), "'") !== $groupId); ++ } ++ ++ /** ++ * Process pending jobs ++ * ++ * @param $groupId ++ * @param $jobsRoot ++ * @param $currentTime ++ */ ++ private function processPendingJobs($groupId, $jobsRoot, $currentTime) ++ { ++ $procesedJobs = []; ++ $pendingJobs = $this->getPendingSchedules($groupId); ++ /** @var \Magento\Cron\Model\Schedule $schedule */ ++ foreach ($pendingJobs as $schedule) { ++ if (isset($procesedJobs[$schedule->getJobCode()])) { ++ // process only on job per run ++ continue; ++ } ++ $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; ++ if (!$jobConfig) { ++ continue; ++ } ++ ++ $scheduledTime = strtotime($schedule->getScheduledAt()); ++ if ($scheduledTime > $currentTime) { ++ continue; ++ } ++ ++ try { ++ if ($schedule->tryLockJob()) { ++ $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); ++ } ++ } catch (\Exception $e) { ++ $schedule->setMessages($e->getMessage()); ++ if ($schedule->getStatus() === Schedule::STATUS_ERROR) { ++ $this->logger->critical($e); ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_MISSED ++ && $this->state->getMode() === State::MODE_DEVELOPER ++ ) { ++ $this->logger->info( ++ sprintf( ++ "%s Schedule Id: %s Job Code: %s", ++ $schedule->getMessages(), ++ $schedule->getScheduleId(), ++ $schedule->getJobCode() ++ ) ++ ); ++ } ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_SUCCESS) { ++ $procesedJobs[$schedule->getJobCode()] = true; ++ } ++ $schedule->save(); ++ } ++ } + } diff --git a/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.4.patch b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.4.patch new file mode 100644 index 0000000..95adfa9 --- /dev/null +++ b/patches/MAGECLOUD-1607__overhaul_cron_implementation__2.2.4.patch @@ -0,0 +1,601 @@ +diff -Naur a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +--- a/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php ++++ b/vendor/magento/module-cron/Observer/ProcessCronQueueObserver.php +@@ -13,6 +13,8 @@ use Magento\Framework\App\State; + use Magento\Framework\Console\Cli; + use Magento\Framework\Event\ObserverInterface; + use \Magento\Cron\Model\Schedule; ++use Magento\Framework\Profiler\Driver\Standard\Stat; ++use Magento\Framework\Profiler\Driver\Standard\StatFactory; + + /** + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) +@@ -126,6 +128,11 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + private $jobs; + ++ /** ++ * @var Stat ++ */ ++ private $statProfiler; ++ + /** + * @param \Magento\Framework\ObjectManagerInterface $objectManager + * @param \Magento\Cron\Model\ScheduleFactory $scheduleFactory +@@ -138,6 +145,7 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory + * @param \Psr\Log\LoggerInterface $logger + * @param \Magento\Framework\App\State $state ++ * @param StatFactory $statFactory + * @SuppressWarnings(PHPMD.ExcessiveParameterList) + */ + public function __construct( +@@ -151,7 +159,8 @@ class ProcessCronQueueObserver implements ObserverInterface + \Magento\Framework\Stdlib\DateTime\DateTime $dateTime, + \Magento\Framework\Process\PhpExecutableFinderFactory $phpExecutableFinderFactory, + \Psr\Log\LoggerInterface $logger, +- \Magento\Framework\App\State $state ++ \Magento\Framework\App\State $state, ++ StatFactory $statFactory + ) { + $this->_objectManager = $objectManager; + $this->_scheduleFactory = $scheduleFactory; +@@ -164,6 +173,7 @@ class ProcessCronQueueObserver implements ObserverInterface + $this->phpExecutableFinder = $phpExecutableFinderFactory->create(); + $this->logger = $logger; + $this->state = $state; ++ $this->statProfiler = $statFactory->create(); + } + + /** +@@ -179,27 +189,26 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + public function execute(\Magento\Framework\Event\Observer $observer) + { +- $pendingJobs = $this->_getPendingSchedules(); ++ + $currentTime = $this->dateTime->gmtTimestamp(); + $jobGroupsRoot = $this->_config->getJobs(); ++ // sort jobs groups to start from used in separated process ++ uksort( ++ $jobGroupsRoot, ++ function ($a, $b) { ++ return $this->getCronGroupConfigurationValue($b, 'use_separate_process') ++ - $this->getCronGroupConfigurationValue($a, 'use_separate_process'); ++ } ++ ); + + $phpPath = $this->phpExecutableFinder->find() ?: 'php'; + + foreach ($jobGroupsRoot as $groupId => $jobsRoot) { +- $this->_cleanup($groupId); +- $this->_generate($groupId); +- if ($this->_request->getParam('group') !== null +- && $this->_request->getParam('group') !== '\'' . ($groupId) . '\'' +- && $this->_request->getParam('group') !== $groupId +- ) { ++ if (!$this->isGroupInFilter($groupId)) { + continue; + } +- if (($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1') && ( +- $this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/use_separate_process', +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ) == 1 +- ) ++ if ($this->_request->getParam(self::STANDALONE_PROCESS_STARTED) !== '1' ++ && $this->getCronGroupConfigurationValue($groupId, 'use_separate_process') == 1 + ) { + $this->_shell->execute( + $phpPath . ' %s cron:run --group=' . $groupId . ' --' . Cli::INPUT_KEY_BOOTSTRAP . '=' +@@ -211,42 +220,9 @@ class ProcessCronQueueObserver implements ObserverInterface + continue; + } + +- /** @var \Magento\Cron\Model\Schedule $schedule */ +- foreach ($pendingJobs as $schedule) { +- $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; +- if (!$jobConfig) { +- continue; +- } +- +- $scheduledTime = strtotime($schedule->getScheduledAt()); +- if ($scheduledTime > $currentTime) { +- continue; +- } +- +- try { +- if ($schedule->tryLockJob()) { +- $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); +- } +- } catch (\Exception $e) { +- $schedule->setMessages($e->getMessage()); +- if ($schedule->getStatus() === Schedule::STATUS_ERROR) { +- $this->logger->critical($e); +- } +- if ($schedule->getStatus() === Schedule::STATUS_MISSED +- && $this->state->getMode() === State::MODE_DEVELOPER +- ) { +- $this->logger->info( +- sprintf( +- "%s Schedule Id: %s Job Code: %s", +- $schedule->getMessages(), +- $schedule->getScheduleId(), +- $schedule->getJobCode() +- ) +- ); +- } +- } +- $schedule->save(); +- } ++ $this->cleanupJobs($groupId, $currentTime); ++ $this->generateSchedules($groupId); ++ $this->processPendingJobs($groupId, $jobsRoot, $currentTime); + } + } + +@@ -263,24 +239,25 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function _runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId) + { +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $jobCode = $schedule->getJobCode(); ++ $scheduleLifetime = $this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_LIFETIME); + $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + if ($scheduledTime < $currentTime - $scheduleLifetime) { + $schedule->setStatus(Schedule::STATUS_MISSED); ++ $this->logger->info(sprintf('Cron Job %s is missed', $jobCode)); + throw new \Exception('Too late for the schedule'); + } + + if (!isset($jobConfig['instance'], $jobConfig['method'])) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception('No callbacks found'); + } + $model = $this->_objectManager->create($jobConfig['instance']); + $callback = [$model, $jobConfig['method']]; + if (!is_callable($callback)) { + $schedule->setStatus(Schedule::STATUS_ERROR); ++ $this->logger->error(sprintf('Cron Job %s has an error', $jobCode)); + throw new \Exception( + sprintf('Invalid callback: %s::%s can\'t be called', $jobConfig['instance'], $jobConfig['method']) + ); +@@ -288,8 +265,12 @@ class ProcessCronQueueObserver implements ObserverInterface + + $schedule->setExecutedAt(strftime('%Y-%m-%d %H:%M:%S', $this->dateTime->gmtTimestamp()))->save(); + ++ $this->startProfiling(); + try { ++ $this->logger->info(sprintf('Cron Job %s is run', $jobCode)); + call_user_func_array($callback, [$schedule]); ++ ++ + } catch (\Throwable $e) { + $schedule->setStatus(Schedule::STATUS_ERROR); + if (!$e instanceof \Exception) { +@@ -299,13 +280,59 @@ class ProcessCronQueueObserver implements ObserverInterface + $e + ); + } ++ $this->logger->error(sprintf( ++ 'Cron Job %s has an error. Statistics: %s %s', ++ $jobCode, ++ $this->getProfilingStat(), $e->getMessage() ++ )); + throw $e; ++ } finally { ++ $this->stopProfiling(); + } + + $schedule->setStatus(Schedule::STATUS_SUCCESS)->setFinishedAt(strftime( + '%Y-%m-%d %H:%M:%S', + $this->dateTime->gmtTimestamp() + )); ++ ++ $this->logger->info(sprintf( ++ 'Cron Job %s is successfully finished. Statistics: %s', ++ $jobCode, ++ $this->getProfilingStat() ++ )); ++ } ++ ++ /** ++ * Starts profiling ++ * ++ * @return void ++ */ ++ private function startProfiling() ++ { ++ $this->statProfiler->clear(); ++ $this->statProfiler->start('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Stops profiling ++ * ++ * @return void ++ */ ++ private function stopProfiling() ++ { ++ $this->statProfiler->stop('job', microtime(true), memory_get_usage(true), memory_get_usage()); ++ } ++ ++ /** ++ * Retrieves statistics in the JSON format ++ * ++ * @return string ++ */ ++ private function getProfilingStat() ++ { ++ $stat = $this->statProfiler->get('job'); ++ unset($stat[Stat::START]); ++ return json_encode($stat); + } + + /** +@@ -313,15 +340,13 @@ class ProcessCronQueueObserver implements ObserverInterface + * + * @return \Magento\Cron\Model\ResourceModel\Schedule\Collection + */ +- protected function _getPendingSchedules() ++ private function getPendingSchedules($groupId) + { +- if (!$this->_pendingSchedules) { +- $this->_pendingSchedules = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- Schedule::STATUS_PENDING +- )->load(); +- } +- return $this->_pendingSchedules; ++ $jobs = $this->getJobs(); ++ $pendingJobs = $this->_scheduleFactory->create()->getCollection(); ++ $pendingJobs->addFieldToFilter('status', Schedule::STATUS_PENDING); ++ $pendingJobs->addFieldToFilter('job_code', ['in' => array_keys($jobs[$groupId])]); ++ return $pendingJobs; + } + + /** +@@ -330,22 +355,32 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param string $groupId + * @return $this + */ +- protected function _generate($groupId) ++ private function generateSchedules($groupId) + { + /** + * check if schedule generation is needed + */ + $lastRun = (int)$this->_cache->load(self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId); +- $rawSchedulePeriod = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_GENERATE_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ $rawSchedulePeriod = (int)$this->getCronGroupConfigurationValue( ++ $groupId, ++ self::XML_PATH_SCHEDULE_GENERATE_EVERY + ); + $schedulePeriod = $rawSchedulePeriod * self::SECONDS_IN_MINUTE; + if ($lastRun > $this->dateTime->gmtTimestamp() - $schedulePeriod) { + return $this; + } + +- $schedules = $this->_getPendingSchedules(); ++ /** ++ * save time schedules generation was ran with no expiration ++ */ ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, ++ ['crontab'], ++ null ++ ); ++ ++ $schedules = $this->getPendingSchedules($groupId); + $exists = []; + /** @var Schedule $schedule */ + foreach ($schedules as $schedule) { +@@ -360,16 +395,6 @@ class ProcessCronQueueObserver implements ObserverInterface + $this->_generateJobs($jobs[$groupId], $exists, $groupId); + $this->cleanupScheduleMismatches(); + +- /** +- * save time schedules generation was ran with no expiration +- */ +- $this->_cache->save( +- $this->dateTime->gmtTimestamp(), +- self::CACHE_KEY_LAST_SCHEDULE_GENERATE_AT . $groupId, +- ['crontab'], +- null +- ); +- + return $this; + } + +@@ -379,7 +404,7 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param array $jobs + * @param array $exists + * @param string $groupId +- * @return $this ++ * @return void + */ + protected function _generateJobs($jobs, $exists, $groupId) + { +@@ -392,77 +417,60 @@ class ProcessCronQueueObserver implements ObserverInterface + $timeInterval = $this->getScheduleTimeInterval($groupId); + $this->saveSchedule($jobCode, $cronExpression, $timeInterval, $exists); + } +- return $this; + } + + /** + * Clean expired jobs + * +- * @param string $groupId +- * @return $this ++ * @param $groupId ++ * @param $currentTime ++ * @return void + */ +- protected function _cleanup($groupId) ++ private function cleanupJobs($groupId, $currentTime) + { +- $this->cleanupDisabledJobs($groupId); +- + // check if history cleanup is needed + $lastCleanup = (int)$this->_cache->load(self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId); +- $historyCleanUp = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_CLEANUP_EVERY, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $historyCleanUp = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_CLEANUP_EVERY); + if ($lastCleanup > $this->dateTime->gmtTimestamp() - $historyCleanUp * self::SECONDS_IN_MINUTE) { + return $this; + } +- +- // check how long the record should stay unprocessed before marked as MISSED +- $scheduleLifetime = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_LIFETIME, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ // save time history cleanup was ran with no expiration ++ $this->_cache->save( ++ $this->dateTime->gmtTimestamp(), ++ self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, ++ ['crontab'], ++ null + ); +- $scheduleLifetime = $scheduleLifetime * self::SECONDS_IN_MINUTE; + +- /** +- * @var \Magento\Cron\Model\ResourceModel\Schedule\Collection $history +- */ +- $history = $this->_scheduleFactory->create()->getCollection()->addFieldToFilter( +- 'status', +- ['in' => [Schedule::STATUS_SUCCESS, Schedule::STATUS_MISSED, Schedule::STATUS_ERROR]] +- )->load(); ++ $this->cleanupDisabledJobs($groupId); + +- $historySuccess = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_SUCCESS, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); +- $historyFailure = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_HISTORY_FAILURE, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $historySuccess = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_SUCCESS); ++ $historyFailure = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_HISTORY_FAILURE); + $historyLifetimes = [ + Schedule::STATUS_SUCCESS => $historySuccess * self::SECONDS_IN_MINUTE, + Schedule::STATUS_MISSED => $historyFailure * self::SECONDS_IN_MINUTE, + Schedule::STATUS_ERROR => $historyFailure * self::SECONDS_IN_MINUTE, ++ Schedule::STATUS_PENDING => max($historyFailure, $historySuccess) * self::SECONDS_IN_MINUTE, + ]; + +- $now = $this->dateTime->gmtTimestamp(); +- /** @var Schedule $record */ +- foreach ($history as $record) { +- $checkTime = $record->getExecutedAt() ? strtotime($record->getExecutedAt()) : +- strtotime($record->getScheduledAt()) + $scheduleLifetime; +- if ($checkTime < $now - $historyLifetimes[$record->getStatus()]) { +- $record->delete(); +- } ++ $jobs = $this->getJobs()[$groupId]; ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $connection = $scheduleResource->getConnection(); ++ $count = 0; ++ foreach ($historyLifetimes as $status => $time) { ++ $count += $connection->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => $status, ++ 'job_code in (?)' => array_keys($jobs), ++ 'created_at < ?' => $connection->formatDate($currentTime - $time) ++ ] ++ ); + } + +- // save time history cleanup was ran with no expiration +- $this->_cache->save( +- $this->dateTime->gmtTimestamp(), +- self::CACHE_KEY_LAST_HISTORY_CLEANUP_AT . $groupId, +- ['crontab'], +- null +- ); +- +- return $this; ++ if ($count) { ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } + } + + /** +@@ -493,7 +501,7 @@ class ProcessCronQueueObserver implements ObserverInterface + for ($time = $currentTime; $time < $timeAhead; $time += self::SECONDS_IN_MINUTE) { + $scheduledAt = strftime('%Y-%m-%d %H:%M:00', $time); + $alreadyScheduled = !empty($exists[$jobCode . '/' . $scheduledAt]); +- $schedule = $this->generateSchedule($jobCode, $cronExpression, $time); ++ $schedule = $this->createSchedule($jobCode, $cronExpression, $time); + $valid = $schedule->trySchedule(); + if (!$valid) { + if ($alreadyScheduled) { +@@ -517,7 +525,7 @@ class ProcessCronQueueObserver implements ObserverInterface + * @param int $time + * @return Schedule + */ +- protected function generateSchedule($jobCode, $cronExpression, $time) ++ protected function createSchedule($jobCode, $cronExpression, $time) + { + $schedule = $this->_scheduleFactory->create() + ->setCronExpr($cronExpression) +@@ -535,10 +543,7 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + protected function getScheduleTimeInterval($groupId) + { +- $scheduleAheadFor = (int)$this->_scopeConfig->getValue( +- 'system/cron/' . $groupId . '/' . self::XML_PATH_SCHEDULE_AHEAD_FOR, +- \Magento\Store\Model\ScopeInterface::SCOPE_STORE +- ); ++ $scheduleAheadFor = (int)$this->getCronGroupConfigurationValue($groupId, self::XML_PATH_SCHEDULE_AHEAD_FOR); + $scheduleAheadFor = $scheduleAheadFor * self::SECONDS_IN_MINUTE; + + return $scheduleAheadFor; +@@ -554,16 +559,26 @@ class ProcessCronQueueObserver implements ObserverInterface + private function cleanupDisabledJobs($groupId) + { + $jobs = $this->getJobs(); ++ $jobsToCleanup = []; + foreach ($jobs[$groupId] as $jobCode => $jobConfig) { + if (!$this->getCronExpression($jobConfig)) { + /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ +- $scheduleResource = $this->_scheduleFactory->create()->getResource(); +- $scheduleResource->getConnection()->delete($scheduleResource->getMainTable(), [ +- 'status=?' => Schedule::STATUS_PENDING, +- 'job_code=?' => $jobCode, +- ]); ++ $jobsToCleanup[] = $jobCode; + } + } ++ ++ if (count($jobsToCleanup) > 0) { ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); ++ $count = $scheduleResource->getConnection()->delete( ++ $scheduleResource->getMainTable(), ++ [ ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code in (?)' => $jobsToCleanup, ++ ] ++ ); ++ ++ $this->logger->info(sprintf('%d cron jobs were cleaned', $count)); ++ } + } + + /** +@@ -593,12 +608,12 @@ class ProcessCronQueueObserver implements ObserverInterface + */ + private function cleanupScheduleMismatches() + { ++ /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ ++ $scheduleResource = $this->_scheduleFactory->create()->getResource(); + foreach ($this->invalid as $jobCode => $scheduledAtList) { +- /** @var \Magento\Cron\Model\ResourceModel\Schedule $scheduleResource */ +- $scheduleResource = $this->_scheduleFactory->create()->getResource(); + $scheduleResource->getConnection()->delete($scheduleResource->getMainTable(), [ +- 'status=?' => Schedule::STATUS_PENDING, +- 'job_code=?' => $jobCode, ++ 'status = ?' => Schedule::STATUS_PENDING, ++ 'job_code = ?' => $jobCode, + 'scheduled_at in (?)' => $scheduledAtList, + ]); + } +@@ -615,4 +630,87 @@ class ProcessCronQueueObserver implements ObserverInterface + } + return $this->jobs; + } ++ ++ /** ++ * Get CronGroup Configuration Value ++ * ++ * @param $groupId ++ * @return int ++ */ ++ private function getCronGroupConfigurationValue($groupId, $path) ++ { ++ return $this->_scopeConfig->getValue( ++ 'system/cron/' . $groupId . '/' . $path, ++ \Magento\Store\Model\ScopeInterface::SCOPE_STORE ++ ); ++ return $scheduleLifetime; ++ } ++ ++ /** ++ * Is Group In Filter ++ * ++ * @param $groupId ++ * @return bool ++ */ ++ private function isGroupInFilter($groupId): bool ++ { ++ return !($this->_request->getParam('group') !== null ++ && trim($this->_request->getParam('group'), "'") !== $groupId); ++ } ++ ++ /** ++ * Process pending jobs ++ * ++ * @param $groupId ++ * @param $jobsRoot ++ * @param $currentTime ++ */ ++ private function processPendingJobs($groupId, $jobsRoot, $currentTime) ++ { ++ $procesedJobs = []; ++ $pendingJobs = $this->getPendingSchedules($groupId); ++ /** @var \Magento\Cron\Model\Schedule $schedule */ ++ foreach ($pendingJobs as $schedule) { ++ if (isset($procesedJobs[$schedule->getJobCode()])) { ++ // process only on job per run ++ continue; ++ } ++ $jobConfig = isset($jobsRoot[$schedule->getJobCode()]) ? $jobsRoot[$schedule->getJobCode()] : null; ++ if (!$jobConfig) { ++ continue; ++ } ++ ++ $scheduledTime = strtotime($schedule->getScheduledAt()); ++ if ($scheduledTime > $currentTime) { ++ continue; ++ } ++ ++ try { ++ if ($schedule->tryLockJob()) { ++ $this->_runJob($scheduledTime, $currentTime, $jobConfig, $schedule, $groupId); ++ } ++ } catch (\Exception $e) { ++ $schedule->setMessages($e->getMessage()); ++ if ($schedule->getStatus() === Schedule::STATUS_ERROR) { ++ $this->logger->critical($e); ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_MISSED ++ && $this->state->getMode() === State::MODE_DEVELOPER ++ ) { ++ $this->logger->info( ++ sprintf( ++ "%s Schedule Id: %s Job Code: %s", ++ $schedule->getMessages(), ++ $schedule->getScheduleId(), ++ $schedule->getJobCode() ++ ) ++ ); ++ } ++ } ++ if ($schedule->getStatus() === Schedule::STATUS_SUCCESS) { ++ $procesedJobs[$schedule->getJobCode()] = true; ++ } ++ $schedule->save(); ++ } ++ } + } diff --git a/patches/MAGECLOUD-1736__respect_minification_override__2.1.4.patch b/patches/MAGECLOUD-1736__respect_minification_override__2.1.4.patch new file mode 100644 index 0000000..3d43d06 --- /dev/null +++ b/patches/MAGECLOUD-1736__respect_minification_override__2.1.4.patch @@ -0,0 +1,23 @@ +diff --git a/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php b/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php +index 82cb8ac51..7997d7cb6 100644 +--- a/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php ++++ b/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php +@@ -107,11 +107,15 @@ class TemplateFile extends File + */ + private function getMinifiedTemplateInProduction($template) + { +- if ($this->deploymentConfig->getConfigData( +- ConfigOptionsListConstants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION +- )) { ++ $forceMinified = $this->deploymentConfig->getConfigData( ++ ConfigOptionsListConstants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION ++ ) ++ || $this->deploymentConfig->getConfigData('force_html_minification'); ++ ++ if ($forceMinified) { + return $this->templateMinifier->getMinified($template); + } ++ + return $this->templateMinifier->getPathToMinified($template); + } + } diff --git a/patches/MAGECLOUD-1736__respect_minification_override__2.2.0.patch b/patches/MAGECLOUD-1736__respect_minification_override__2.2.0.patch new file mode 100644 index 0000000..ee97f39 --- /dev/null +++ b/patches/MAGECLOUD-1736__respect_minification_override__2.2.0.patch @@ -0,0 +1,20 @@ +diff --git a/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php b/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php +index 09f87d878..5ef71afcc 100644 +--- a/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php ++++ b/vendor/magento/framework/View/Design/FileResolution/Fallback/TemplateFile.php +@@ -107,9 +107,12 @@ class TemplateFile extends File + */ + private function getMinifiedTemplateInProduction($template) + { +- if ($this->deploymentConfig->getConfigData( +- ConfigOptionsListConstants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION +- )) { ++ $forceMinified = $this->deploymentConfig->getConfigData( ++ ConfigOptionsListConstants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION ++ ) ++ || $this->deploymentConfig->getConfigData('force_html_minification'); ++ ++ if ($forceMinified) { + return $this->templateMinifier->getMinified($template); + } + return $this->templateMinifier->getPathToMinified($template); diff --git a/patches/MAGECLOUD-1998__unify_sitemapxml_and_robotstxt_generation__2.1.11.patch b/patches/MAGECLOUD-1998__unify_sitemapxml_and_robotstxt_generation__2.1.11.patch new file mode 100644 index 0000000..3b58464 --- /dev/null +++ b/patches/MAGECLOUD-1998__unify_sitemapxml_and_robotstxt_generation__2.1.11.patch @@ -0,0 +1,161 @@ +diff -Naur a/vendor/magento/module-config/Model/Config/Reader/Source/Deployed/DocumentRoot.php b/vendor/magento/module-config/Model/Config/Reader/Source/Deployed/DocumentRoot.php +new file mode 100644 +--- /dev/null ++++ b/vendor/magento/module-config/Model/Config/Reader/Source/Deployed/DocumentRoot.php +@@ -0,0 +1,55 @@ ++config = $config; ++ } ++ ++ /** ++ * A shortcut to load the document root path from the DirectoryList based on the ++ * deployment configuration. ++ * ++ * @return string ++ */ ++ public function getPath() ++ { ++ return $this->isPub() ? DirectoryList::PUB : DirectoryList::ROOT; ++ } ++ ++ /** ++ * Returns whether the deployment configuration specifies that the document root is ++ * in the pub/ folder. This affects ares such as sitemaps and robots.txt (and will ++ * likely be extended to control other areas). ++ * ++ * @return bool ++ */ ++ public function isPub() ++ { ++ return (bool)$this->config->get(ConfigOptionsListConstants::CONFIG_PATH_DOCUMENT_ROOT_IS_PUB); ++ } ++} +diff -Naur a/vendor/magento/module-sitemap/Block/Adminhtml/Grid/Renderer/Link.php b/vendor/magento/module-sitemap/Block/Adminhtml/Grid/Renderer/Link.php +--- a/vendor/magento/module-sitemap/Block/Adminhtml/Grid/Renderer/Link.php ++++ b/vendor/magento/module-sitemap/Block/Adminhtml/Grid/Renderer/Link.php +@@ -11,6 +11,8 @@ + namespace Magento\Sitemap\Block\Adminhtml\Grid\Renderer; + + use Magento\Framework\App\Filesystem\DirectoryList; ++use Magento\Config\Model\Config\Reader\Source\Deployed\DocumentRoot; ++use Magento\Framework\App\ObjectManager; + + class Link extends \Magento\Backend\Block\Widget\Grid\Column\Renderer\AbstractRenderer + { +@@ -25,19 +27,28 @@ class Link extends \Magento\Backend\Block\Widget\Grid\Column\Renderer\AbstractRe + protected $_sitemapFactory; + + /** ++ * @var DocumentRoot ++ */ ++ protected $documentRoot; ++ ++ /** + * @param \Magento\Backend\Block\Context $context + * @param \Magento\Sitemap\Model\SitemapFactory $sitemapFactory + * @param \Magento\Framework\Filesystem $filesystem + * @param array $data ++ * @param DocumentRoot $documentRoot + */ + public function __construct( + \Magento\Backend\Block\Context $context, + \Magento\Sitemap\Model\SitemapFactory $sitemapFactory, + \Magento\Framework\Filesystem $filesystem, +- array $data = [] ++ array $data = [], ++ DocumentRoot $documentRoot = null + ) { + $this->_sitemapFactory = $sitemapFactory; + $this->_filesystem = $filesystem; ++ $this->documentRoot = $documentRoot ?: ObjectManager::getInstance()->get(DocumentRoot::class); ++ + parent::__construct($context, $data); + } + +@@ -54,7 +65,8 @@ class Link extends \Magento\Backend\Block\Widget\Grid\Column\Renderer\AbstractRe + $url = $this->escapeHtml($sitemap->getSitemapUrl($row->getSitemapPath(), $row->getSitemapFilename())); + + $fileName = preg_replace('/^\//', '', $row->getSitemapPath() . $row->getSitemapFilename()); +- $directory = $this->_filesystem->getDirectoryRead(DirectoryList::ROOT); ++ $documentRootPath = $this->documentRoot->getPath(); ++ $directory = $this->_filesystem->getDirectoryRead($documentRootPath); + if ($directory->isFile($fileName)) { + return sprintf('%1$s', $url); + } +diff -Naur a/vendor/magento/module-sitemap/Model/Sitemap.php b/vendor/magento/module-sitemap/Model/Sitemap.php +--- a/vendor/magento/module-sitemap/Model/Sitemap.php ++++ b/vendor/magento/module-sitemap/Model/Sitemap.php +@@ -8,7 +8,8 @@ + + namespace Magento\Sitemap\Model; + +-use Magento\Framework\App\Filesystem\DirectoryList; ++use Magento\Config\Model\Config\Reader\Source\Deployed\DocumentRoot; ++use Magento\Framework\App\ObjectManager; + use Magento\Robots\Model\Config\Value; + + /** +@@ -170,6 +171,7 @@ + * @param \Magento\Framework\Model\ResourceModel\AbstractResource $resource + * @param \Magento\Framework\Data\Collection\AbstractDb $resourceCollection + * @param array $data ++ * @param DocumentRoot|null $documentRoot + * @SuppressWarnings(PHPMD.ExcessiveParameterList) + */ + public function __construct( +@@ -187,11 +189,13 @@ + \Magento\Framework\Stdlib\DateTime $dateTime, + \Magento\Framework\Model\ResourceModel\AbstractResource $resource = null, + \Magento\Framework\Data\Collection\AbstractDb $resourceCollection = null, +- array $data = [] ++ array $data = [], ++ \Magento\Config\Model\Config\Reader\Source\Deployed\DocumentRoot $documentRoot = null + ) { + $this->_escaper = $escaper; + $this->_sitemapData = $sitemapData; +- $this->_directory = $filesystem->getDirectoryWrite(DirectoryList::ROOT); ++ $documentRoot = $documentRoot ?: ObjectManager::getInstance()->get(DocumentRoot::class); ++ $this->_directory = $filesystem->getDirectoryWrite($documentRoot->getPath()); + $this->_categoryFactory = $categoryFactory; + $this->_productFactory = $productFactory; + $this->_cmsFactory = $cmsFactory; +diff -Naur a/vendor/magento/framework/Config/ConfigOptionsListConstants.php b/vendor/magento/framework/Config/ConfigOptionsListConstants.php +--- a/vendor/magento/framework/Config/ConfigOptionsListConstants.php ++++ b/vendor/magento/framework/Config/ConfigOptionsListConstants.php +@@ -30,6 +30,8 @@ class ConfigOptionsListConstants + const CONFIG_PATH_DB = 'db'; + const CONFIG_PATH_RESOURCE = 'resource'; + const CONFIG_PATH_CACHE_TYPES = 'cache_types'; ++ const CONFIG_PATH_DOCUMENT_ROOT_IS_PUB = 'directories/document_root_is_pub'; ++ + /**#@-*/ + + /**#@+ diff --git a/patches/MAGECLOUD-1998__unify_sitemapxml_and_robotstxt_generation__2.1.4.patch b/patches/MAGECLOUD-1998__unify_sitemapxml_and_robotstxt_generation__2.1.4.patch new file mode 100644 index 0000000..06bc09e --- /dev/null +++ b/patches/MAGECLOUD-1998__unify_sitemapxml_and_robotstxt_generation__2.1.4.patch @@ -0,0 +1,165 @@ +diff -Naur a/vendor/magento/module-config/Model/Config/Backend/Admin/Robots.php b/vendor/magento/module-config/Model/Config/Backend/Admin/Robots.php +--- a/vendor/magento/module-config/Model/Config/Backend/Admin/Robots.php ++++ b/vendor/magento/module-config/Model/Config/Backend/Admin/Robots.php +@@ -44,7 +44,7 @@ + array $data = [] + ) { + parent::__construct($context, $registry, $config, $cacheTypeList, $resource, $resourceCollection, $data); +- $this->_directory = $filesystem->getDirectoryWrite(DirectoryList::ROOT); ++ $this->_directory = $filesystem->getDirectoryWrite(DirectoryList::MEDIA); + $this->_file = 'robots.txt'; + } + +diff -Naur a/vendor/magento/module-config/Model/Config/Reader/Source/Deployed/DocumentRoot.php b/vendor/magento/module-config/Model/Config/Reader/Source/Deployed/DocumentRoot.php +new file mode 100644 +--- /dev/null ++++ b/vendor/magento/module-config/Model/Config/Reader/Source/Deployed/DocumentRoot.php +@@ -0,0 +1,55 @@ ++config = $config; ++ } ++ ++ /** ++ * A shortcut to load the document root path from the DirectoryList based on the ++ * deployment configuration. ++ * ++ * @return string ++ */ ++ public function getPath() ++ { ++ return $this->isPub() ? DirectoryList::PUB : DirectoryList::ROOT; ++ } ++ ++ /** ++ * Returns whether the deployment configuration specifies that the document root is ++ * in the pub/ folder. This affects ares such as sitemaps and robots.txt (and will ++ * likely be extended to control other areas). ++ * ++ * @return bool ++ */ ++ public function isPub() ++ { ++ return (bool)$this->config->get(ConfigOptionsListConstants::CONFIG_PATH_DOCUMENT_ROOT_IS_PUB); ++ } ++} +diff -Naur a/vendor/magento/module-sitemap/Block/Adminhtml/Grid/Renderer/Link.php b/vendor/magento/module-sitemap/Block/Adminhtml/Grid/Renderer/Link.php +--- a/vendor/magento/module-sitemap/Block/Adminhtml/Grid/Renderer/Link.php ++++ b/vendor/magento/module-sitemap/Block/Adminhtml/Grid/Renderer/Link.php +@@ -11,6 +11,8 @@ + namespace Magento\Sitemap\Block\Adminhtml\Grid\Renderer; + + use Magento\Framework\App\Filesystem\DirectoryList; ++use Magento\Config\Model\Config\Reader\Source\Deployed\DocumentRoot; ++use Magento\Framework\App\ObjectManager; + + class Link extends \Magento\Backend\Block\Widget\Grid\Column\Renderer\AbstractRenderer + { +@@ -25,19 +27,28 @@ class Link extends \Magento\Backend\Block\Widget\Grid\Column\Renderer\AbstractRe + protected $_sitemapFactory; + + /** ++ * @var DocumentRoot ++ */ ++ protected $documentRoot; ++ ++ /** + * @param \Magento\Backend\Block\Context $context + * @param \Magento\Sitemap\Model\SitemapFactory $sitemapFactory + * @param \Magento\Framework\Filesystem $filesystem + * @param array $data ++ * @param DocumentRoot $documentRoot + */ + public function __construct( + \Magento\Backend\Block\Context $context, + \Magento\Sitemap\Model\SitemapFactory $sitemapFactory, + \Magento\Framework\Filesystem $filesystem, +- array $data = [] ++ array $data = [], ++ DocumentRoot $documentRoot = null + ) { + $this->_sitemapFactory = $sitemapFactory; + $this->_filesystem = $filesystem; ++ $this->documentRoot = $documentRoot ?: ObjectManager::getInstance()->get(DocumentRoot::class); ++ + parent::__construct($context, $data); + } + +@@ -54,7 +65,8 @@ class Link extends \Magento\Backend\Block\Widget\Grid\Column\Renderer\AbstractRe + $url = $this->escapeHtml($sitemap->getSitemapUrl($row->getSitemapPath(), $row->getSitemapFilename())); + + $fileName = preg_replace('/^\//', '', $row->getSitemapPath() . $row->getSitemapFilename()); +- $directory = $this->_filesystem->getDirectoryRead(DirectoryList::ROOT); ++ $documentRootPath = $this->documentRoot->getPath(); ++ $directory = $this->_filesystem->getDirectoryRead($documentRootPath); + if ($directory->isFile($fileName)) { + return sprintf('%1$s', $url); + } +diff -Naur a/vendor/magento/module-sitemap/Model/Sitemap.php b/vendor/magento/module-sitemap/Model/Sitemap.php +--- a/vendor/magento/module-sitemap/Model/Sitemap.php ++++ b/vendor/magento/module-sitemap/Model/Sitemap.php +@@ -8,7 +8,8 @@ + + namespace Magento\Sitemap\Model; + +-use Magento\Framework\App\Filesystem\DirectoryList; ++use Magento\Config\Model\Config\Reader\Source\Deployed\DocumentRoot; ++use Magento\Framework\App\ObjectManager; + + /** + * Sitemap model +@@ -179,11 +180,13 @@ class Sitemap extends \Magento\Framework\Model\AbstractModel + \Magento\Framework\Stdlib\DateTime $dateTime, + \Magento\Framework\Model\ResourceModel\AbstractResource $resource = null, + \Magento\Framework\Data\Collection\AbstractDb $resourceCollection = null, +- array $data = [] ++ array $data = [], ++ DocumentRoot $documentRoot = null + ) { + $this->_escaper = $escaper; + $this->_sitemapData = $sitemapData; +- $this->_directory = $filesystem->getDirectoryWrite(DirectoryList::ROOT); ++ $documentRoot = $documentRoot ?: ObjectManager::getInstance()->get(DocumentRoot::class); ++ $this->_directory = $filesystem->getDirectoryWrite($documentRoot->getPath()); + $this->_categoryFactory = $categoryFactory; + $this->_productFactory = $productFactory; + $this->_cmsFactory = $cmsFactory; +diff -Naur a/vendor/magento/framework/Config/ConfigOptionsListConstants.php b/vendor/magento/framework/Config/ConfigOptionsListConstants.php +--- a/vendor/magento/framework/Config/ConfigOptionsListConstants.php ++++ b/vendor/magento/framework/Config/ConfigOptionsListConstants.php +@@ -30,6 +30,8 @@ class ConfigOptionsListConstants + const CONFIG_PATH_DB = 'db'; + const CONFIG_PATH_RESOURCE = 'resource'; + const CONFIG_PATH_CACHE_TYPES = 'cache_types'; ++ const CONFIG_PATH_DOCUMENT_ROOT_IS_PUB = 'directories/document_root_is_pub'; ++ + /**#@-*/ + + /**#@+ diff --git a/patches/MAGECLOUD-2033__prevent_deadlock_during_db_dump__2.2.0.patch b/patches/MAGECLOUD-2033__prevent_deadlock_during_db_dump__2.2.0.patch new file mode 100644 index 0000000..4302704 --- /dev/null +++ b/patches/MAGECLOUD-2033__prevent_deadlock_during_db_dump__2.2.0.patch @@ -0,0 +1,13 @@ +diff --git a/vendor/magento/module-support/Console/Command/AbstractBackupDumpCommand.php b/vendor/magento/module-support/Console/Command/AbstractBackupDumpCommand.php +index 673d65ec1b2..aa198e265b0 100644 +--- a/vendor/magento/module-support/Console/Command/AbstractBackupDumpCommand.php ++++ b/vendor/magento/module-support/Console/Command/AbstractBackupDumpCommand.php +@@ -181,7 +181,7 @@ class AbstractBackupDumpCommand extends AbstractBackupCommand + : $this->getParam(ConfigOptionsListConstants::KEY_HOST); + + $this->dbConnectionParams = sprintf( +- '-u%s -h%s %s %s %s', ++ '-u%s -h%s %s %s %s %s', + $this->getParam(ConfigOptionsListConstants::KEY_USER), + $host, + $port, diff --git a/patches/MAGECLOUD-2159__unlock_locale_editing_when_scd_on_demand__2.2.0.patch b/patches/MAGECLOUD-2159__unlock_locale_editing_when_scd_on_demand__2.2.0.patch new file mode 100644 index 0000000..7d62c39 --- /dev/null +++ b/patches/MAGECLOUD-2159__unlock_locale_editing_when_scd_on_demand__2.2.0.patch @@ -0,0 +1,345 @@ +diff -Nuar a/vendor/magento/module-backend/etc/adminhtml/di.xml b/vendor/magento/module-backend/etc/adminhtml/di.xml +--- a/vendor/magento/module-backend/etc/adminhtml/di.xml ++++ b/vendor/magento/module-backend/etc/adminhtml/di.xml +@@ -139,10 +139,16 @@ + + + +- ++ + + + Magento\Config\Model\Config\Structure\ElementVisibilityInterface::HIDDEN ++ ++ ++ ++ ++ ++ + Magento\Config\Model\Config\Structure\ElementVisibilityInterface::DISABLED + + + +diff -Nuar a/vendor/magento/module-config/etc/adminhtml/di.xml b/vendor/magento/module-config/etc/adminhtml/di.xml +--- a/vendor/magento/module-config/etc/adminhtml/di.xml ++++ b/vendor/magento/module-config/etc/adminhtml/di.xml +@@ -15,6 +15,8 @@ + + + Magento\Config\Model\Config\Structure\ConcealInProductionConfigList ++ Magento\Config\Model\Config\Structure\ElementVisibility\ConcealInProduction ++ Magento\Config\Model\Config\Structure\ElementVisibility\ConcealInProductionWithoutScdOnDemand + + + + +diff -Nuar a/vendor/magento/framework/Locale/Deployed/Options.php b/vendor/magento/framework/Locale/Deployed/Options.php +--- a/vendor/magento/framework/Locale/Deployed/Options.php ++++ b/vendor/magento/framework/Locale/Deployed/Options.php +@@ -3,9 +3,14 @@ + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ ++declare(strict_types=1); ++ + namespace Magento\Framework\Locale\Deployed; + ++use Magento\Framework\App\DeploymentConfig; ++use Magento\Framework\App\ObjectManager; + use Magento\Framework\App\State; ++use Magento\Framework\Config\ConfigOptionsListConstants as Constants; + use Magento\Framework\Exception\LocalizedException; + use Magento\Framework\Locale\AvailableLocalesInterface; + use Magento\Framework\Locale\ListsInterface; +@@ -45,28 +50,36 @@ class Options implements OptionInterface + */ + private $localeLists; + ++ /** ++ * @var DeploymentConfig ++ */ ++ private $deploymentConfig; ++ + /** + * @param ListsInterface $localeLists locales list + * @param State $state application state class + * @param AvailableLocalesInterface $availableLocales operates with available locales + * @param DesignInterface $design operates with magento design settings ++ * @param DeploymentConfig $deploymentConfig + */ + public function __construct( + ListsInterface $localeLists, + State $state, + AvailableLocalesInterface $availableLocales, +- DesignInterface $design ++ DesignInterface $design, ++ DeploymentConfig $deploymentConfig = null + ) { + $this->localeLists = $localeLists; + $this->state = $state; + $this->availableLocales = $availableLocales; + $this->design = $design; ++ $this->deploymentConfig = $deploymentConfig ?: ObjectManager::getInstance()->get(DeploymentConfig::class); + } + + /** + * {@inheritdoc} + */ +- public function getOptionLocales() ++ public function getOptionLocales(): array + { + return $this->filterLocales($this->localeLists->getOptionLocales()); + } +@@ -74,7 +87,7 @@ class Options implements OptionInterface + /** + * {@inheritdoc} + */ +- public function getTranslatedOptionLocales() ++ public function getTranslatedOptionLocales(): array + { + return $this->filterLocales($this->localeLists->getTranslatedOptionLocales()); + } +@@ -82,7 +95,7 @@ class Options implements OptionInterface + /** + * Filter list of locales by available locales for current theme and depends on running application mode. + * +- * Applies filters only in production mode. ++ * Applies filters only in production mode when flag 'static_content_on_demand_in_production' is not enabled. + * For example, if the current design theme has only one generated locale en_GB then for given array of locales: + * ```php + * $locales = [ +@@ -113,9 +126,10 @@ class Options implements OptionInterface + * @param array $locales list of locales for filtering + * @return array of filtered locales + */ +- private function filterLocales(array $locales) ++ private function filterLocales(array $locales): array + { +- if ($this->state->getMode() != State::MODE_PRODUCTION) { ++ if ($this->state->getMode() != State::MODE_PRODUCTION ++ || $this->deploymentConfig->getConfigData(Constants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION)) { + return $locales; + } + + +diff -Nuar a/vendor/magento/module-config/Model/Config/Structure/ElementVisibility/ConcealInProduction.php b/vendor/magento/module-config/Model/Config/Structure/ElementVisibility/ConcealInProduction.php +new file mode 100644 +--- /dev/null ++++ b/vendor/magento/module-config/Model/Config/Structure/ElementVisibility/ConcealInProduction.php +@@ -0,0 +1,138 @@ ++ Settings > Configuration page ++ * in Admin Panel in Production mode. ++ * @api ++ */ ++class ConcealInProduction implements ElementVisibilityInterface ++{ ++ /** ++ * The list of form element paths with concrete visibility status. ++ * ++ * E.g. ++ * ++ * ```php ++ * [ ++ * 'general/locale/code' => ElementVisibilityInterface::DISABLED, ++ * 'general/country' => ElementVisibilityInterface::HIDDEN, ++ * ]; ++ * ``` ++ * ++ * It means that: ++ * - field Locale (in group Locale Options in section General) will be disabled ++ * - group Country Options (in section General) will be hidden ++ * ++ * @var array ++ */ ++ private $configs = []; ++ ++ /** ++ * The object that has information about the state of the system. ++ * ++ * @var State ++ */ ++ private $state; ++ ++ /** ++ * ++ * The list of form element paths which ignore visibility status. ++ * ++ * E.g. ++ * ++ * ```php ++ * [ ++ * 'general/country/default' => '', ++ * ]; ++ * ``` ++ * ++ * It means that: ++ * - field 'default' in group Country Options (in section General) will be showed, even if all group(section) ++ * will be hidden. ++ * ++ * @var array ++ */ ++ private $exemptions = []; ++ ++ /** ++ * @param State $state The object that has information about the state of the system ++ * @param array $configs The list of form element paths with concrete visibility status. ++ * @param array $exemptions The list of form element paths which ignore visibility status. ++ */ ++ public function __construct(State $state, array $configs = [], array $exemptions = []) ++ { ++ $this->state = $state; ++ $this->configs = $configs; ++ $this->exemptions = $exemptions; ++ } ++ ++ /** ++ * @inheritdoc ++ * @since 100.2.0 ++ */ ++ public function isHidden($path) ++ { ++ $path = $this->normalizePath($path); ++ if ($this->state->getMode() === State::MODE_PRODUCTION ++ && preg_match('/(?(?
.*?)\/.*?)\/.*?/', $path, $match)) { ++ $group = $match['group']; ++ $section = $match['section']; ++ $exemptions = array_keys($this->exemptions); ++ $checkedItems = []; ++ foreach ([$path, $group, $section] as $itemPath) { ++ $checkedItems[] = $itemPath; ++ if (!empty($this->configs[$itemPath])) { ++ return $this->configs[$itemPath] === static::HIDDEN ++ && empty(array_intersect($checkedItems, $exemptions)); ++ } ++ } ++ } ++ ++ return false; ++ } ++ ++ /** ++ * @inheritdoc ++ * @since 100.2.0 ++ */ ++ public function isDisabled($path) ++ { ++ $path = $this->normalizePath($path); ++ if ($this->state->getMode() === State::MODE_PRODUCTION) { ++ while (true) { ++ if (!empty($this->configs[$path])) { ++ return $this->configs[$path] === static::DISABLED; ++ } ++ ++ $position = strripos($path, '/'); ++ if ($position === false) { ++ break; ++ } ++ $path = substr($path, 0, $position); ++ } ++ } ++ ++ return false; ++ } ++ ++ /** ++ * Returns normalized path. ++ * ++ * @param string $path The path to be normalized ++ * @return string The normalized path ++ */ ++ private function normalizePath($path) ++ { ++ return trim($path, '/'); ++ } ++} + +diff --git a/vendor/magento/module-config/Model/Config/Structure/ElementVisibility/ConcealInProductionWithoutScdOnDemand.php b/vendor/magento/module-config/Model/Config/Structure/ElementVisibility/ConcealInProductionWithoutScdOnDemand.php +new file mode 100644 +--- /dev/null ++++ b/vendor/magento/module-config/Model/Config/Structure/ElementVisibility/ConcealInProductionWithoutScdOnDemand.php +@@ -0,0 +1,72 @@ ++ Settings > Configuration page ++ * when Constants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION is enabled ++ * otherwise rule from Magento\Config\Model\Config\Structure\ElementVisibility\ConcealInProduction is used ++ * @see \Magento\Config\Model\Config\Structure\ElementVisibility\ConcealInProduction ++ * ++ * @api ++ */ ++class ConcealInProductionWithoutScdOnDemand implements ElementVisibilityInterface ++{ ++ /** ++ * @var ConcealInProduction Element visibility rules in the Production mode ++ */ ++ private $concealInProduction; ++ ++ /** ++ * @var DeploymentConfig The application deployment configuration ++ */ ++ private $deploymentConfig; ++ ++ /** ++ * @param ConcealInProductionFactory $concealInProductionFactory ++ * @param DeploymentConfig $deploymentConfig Deployment configuration reader ++ * @param array $configs The list of form element paths with concrete visibility status. ++ * @param array $exemptions The list of form element paths which ignore visibility status. ++ */ ++ public function __construct( ++ ConcealInProductionFactory $concealInProductionFactory, ++ DeploymentConfig $deploymentConfig, ++ array $configs = [], ++ array $exemptions = [] ++ ) { ++ $this->concealInProduction = $concealInProductionFactory ++ ->create(['configs' => $configs, 'exemptions' => $exemptions]); ++ $this->deploymentConfig = $deploymentConfig; ++ } ++ ++ /** ++ * @inheritdoc ++ */ ++ public function isHidden($path): bool ++ { ++ if (!$this->deploymentConfig->getConfigData(Constants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION)) { ++ return $this->concealInProduction->isHidden($path); ++ } ++ return false; ++ } ++ ++ /** ++ * @inheritdoc ++ */ ++ public function isDisabled($path): bool ++ { ++ if (!$this->deploymentConfig->getConfigData(Constants::CONFIG_PATH_SCD_ON_DEMAND_IN_PRODUCTION)) { ++ return $this->concealInProduction->isDisabled($path); ++ } ++ return false; ++ } ++} diff --git a/patches/MAGECLOUD-2173__the_recursion_error_during_deployment__2.2.0.patch b/patches/MAGECLOUD-2173__the_recursion_error_during_deployment__2.2.0.patch new file mode 100644 index 0000000..775982f --- /dev/null +++ b/patches/MAGECLOUD-2173__the_recursion_error_during_deployment__2.2.0.patch @@ -0,0 +1,16 @@ +diff -Nuar a/vendor/magento/module-config/App/Config/Type/System.php b/vendor/magento/module-config/App/Config/Type/System.php +--- a/vendor/magento/module-config/App/Config/Type/System.php ++++ b/vendor/magento/module-config/App/Config/Type/System.php +@@ -155,7 +155,11 @@ class System implements ConfigTypeInterface + } + $scopeId = array_shift($pathParts); + if (!isset($this->data[$scopeType][$scopeId])) { +- $this->data = array_replace_recursive($this->loadScopeData($scopeType, $scopeId), $this->data); ++ $scopeData = $this->loadScopeData($scopeType, $scopeId); ++ /* Starting from 2.2.0 $this->data can be already loaded with $this->loadScopeData */ ++ if (!isset($this->data[$scopeType][$scopeId])) { ++ $this->data = array_replace_recursive($scopeData, $this->data); ++ } + } + return isset($this->data[$scopeType][$scopeId]) + ? $this->getDataByPathParts($this->data[$scopeType][$scopeId], $pathParts) diff --git a/patches/MAGECLOUD-2209__write_logs_for_failed_process_of_generating_factories_in_extensions__2.2.0.patch b/patches/MAGECLOUD-2209__write_logs_for_failed_process_of_generating_factories_in_extensions__2.2.0.patch new file mode 100644 index 0000000..9228a64 --- /dev/null +++ b/patches/MAGECLOUD-2209__write_logs_for_failed_process_of_generating_factories_in_extensions__2.2.0.patch @@ -0,0 +1,159 @@ +diff -Naur a/vendor/magento/framework/Code/Generator.php b/vendor/magento/framework/Code/Generator.php + +--- a/vendor/magento/framework/Code/Generator.php ++++ b/vendor/magento/framework/Code/Generator.php +@@ -7,6 +7,11 @@ namespace Magento\Framework\Code; + + use Magento\Framework\Code\Generator\DefinedClasses; + use Magento\Framework\Code\Generator\EntityAbstract; ++use Magento\Framework\Code\Generator\Io; ++use Magento\Framework\ObjectManagerInterface; ++use Magento\Framework\Phrase; ++use Magento\Framework\Filesystem\Driver\File; ++use Psr\Log\LoggerInterface; + + class Generator + { +@@ -17,7 +22,7 @@ class Generator + const GENERATION_SKIP = 'skip'; + + /** +- * @var \Magento\Framework\Code\Generator\Io ++ * @var Io + */ + protected $_ioObject; + +@@ -32,26 +37,33 @@ class Generator + protected $definedClasses; + + /** +- * @var \Magento\Framework\ObjectManagerInterface ++ * @var ObjectManagerInterface + */ + protected $objectManager; + + /** +- * @param Generator\Io $ioObject +- * @param array $generatedEntities ++ * Logger instance ++ * ++ * @var LoggerInterface ++ */ ++ private $logger; ++ ++ /** ++ * @param Generator\Io $ioObject ++ * @param array $generatedEntities + * @param DefinedClasses $definedClasses ++ * @param LoggerInterface $logger + */ + public function __construct( +- \Magento\Framework\Code\Generator\Io $ioObject = null, ++ Io $ioObject = null, + array $generatedEntities = [], +- DefinedClasses $definedClasses = null ++ DefinedClasses $definedClasses = null, ++ LoggerInterface $logger = null + ) { +- $this->_ioObject = $ioObject +- ?: new \Magento\Framework\Code\Generator\Io( +- new \Magento\Framework\Filesystem\Driver\File() +- ); ++ $this->_ioObject = $ioObject ?: new Io(new File()); + $this->definedClasses = $definedClasses ?: new DefinedClasses(); + $this->_generatedEntities = $generatedEntities; ++ $this->logger = $logger; + } + + /** +@@ -111,8 +123,16 @@ class Generator + if ($generator !== null) { + $this->tryToLoadSourceClass($className, $generator); + if (!($file = $generator->generate())) { ++ /** @var $logger LoggerInterface */ + $errors = $generator->getErrors(); +- throw new \RuntimeException(implode(' ', $errors) . ' in [' . $className . ']'); ++ $errors[] = 'Class ' . $className . ' generation error: The requested class did not generate properly, ' ++ . 'because the \'generated\' directory permission is read-only. ' ++ . 'If --- after running the \'bin/magento setup:di:compile\' CLI command when the \'generated\' ' ++ . 'directory permission is set to write --- the requested class did not generate properly, then ' ++ . 'you must add the generated class object to the signature of the related construct method, only.'; ++ $message = implode(PHP_EOL, $errors); ++ $this->getLogger()->critical($message); ++ throw new \RuntimeException($message); + } + if (!$this->definedClasses->isClassLoadableFromMemory($className)) { + $this->_ioObject->includeFile($file); +@@ -121,13 +141,26 @@ class Generator + } + } + ++ /** ++ * Retrieve logger ++ * ++ * @return LoggerInterface ++ */ ++ private function getLogger() ++ { ++ if (!$this->logger) { ++ $this->logger = $this->getObjectManager()->get(LoggerInterface::class); ++ } ++ return $this->logger; ++ } ++ + /** + * Create entity generator + * + * @param string $generatorClass + * @param string $entityName + * @param string $className +- * @return \Magento\Framework\Code\Generator\EntityAbstract ++ * @return EntityAbstract + */ + protected function createGeneratorInstance($generatorClass, $entityName, $className) + { +@@ -140,10 +173,10 @@ class Generator + /** + * Set object manager instance. + * +- * @param \Magento\Framework\ObjectManagerInterface $objectManager ++ * @param ObjectManagerInterface $objectManager + * @return $this + */ +- public function setObjectManager(\Magento\Framework\ObjectManagerInterface $objectManager) ++ public function setObjectManager(ObjectManagerInterface $objectManager) + { + $this->objectManager = $objectManager; + return $this; +@@ -152,11 +185,11 @@ class Generator + /** + * Get object manager instance. + * +- * @return \Magento\Framework\ObjectManagerInterface ++ * @return ObjectManagerInterface + */ + public function getObjectManager() + { +- if (!($this->objectManager instanceof \Magento\Framework\ObjectManagerInterface)) { ++ if (!($this->objectManager instanceof ObjectManagerInterface)) { + throw new \LogicException( + "Object manager was expected to be set using setObjectManger() " + . "before getObjectManager() invocation." +@@ -169,7 +202,7 @@ class Generator + * Try to load/generate source class to check if it is valid or not. + * + * @param string $className +- * @param \Magento\Framework\Code\Generator\EntityAbstract $generator ++ * @param EntityAbstract $generator + * @return void + * @throws \RuntimeException + */ +@@ -178,7 +211,7 @@ class Generator + $sourceClassName = $generator->getSourceClassName(); + if (!$this->definedClasses->isClassLoadable($sourceClassName)) { + if ($this->generateClass($sourceClassName) !== self::GENERATION_SUCCESS) { +- $phrase = new \Magento\Framework\Phrase( ++ $phrase = new Phrase( + 'Source class "%1" for "%2" generation does not exist.', + [$sourceClassName, $className] + ); diff --git a/patches/MAGECLOUD-2427__resolve_issues_with_cron_schedule.patch b/patches/MAGECLOUD-2427__resolve_issues_with_cron_schedule.patch new file mode 100644 index 0000000..b289920 --- /dev/null +++ b/patches/MAGECLOUD-2427__resolve_issues_with_cron_schedule.patch @@ -0,0 +1,28 @@ +diff -Nuar a/vendor/magento/module-cron/etc/cron_groups.xml b/vendor/magento/module-cron/etc/cron_groups.xml +--- a/vendor/magento/module-cron/etc/cron_groups.xml ++++ b/vendor/magento/module-cron/etc/cron_groups.xml +@@ -11,8 +11,8 @@ + 20 + 15 + 10 +- 10080 +- 10080 ++ 60 ++ 4320 + 0 + + +diff -Nuar a/vendor/magento/module-indexer/etc/cron_groups.xml b/vendor/magento/module-indexer/etc/cron_groups.xml +--- a/vendor/magento/module-indexer/etc/cron_groups.xml ++++ b/vendor/magento/module-indexer/etc/cron_groups.xml +@@ -11,8 +11,8 @@ + 4 + 2 + 10 +- 10080 +- 10080 ++ 60 ++ 4320 + 1 + + diff --git a/patches/MAGECLOUD-2445__do_not_run_cron_when_it_is_disabled__2.1.4.patch b/patches/MAGECLOUD-2445__do_not_run_cron_when_it_is_disabled__2.1.4.patch new file mode 100644 index 0000000..89817a0 --- /dev/null +++ b/patches/MAGECLOUD-2445__do_not_run_cron_when_it_is_disabled__2.1.4.patch @@ -0,0 +1,62 @@ +diff -Nuar a/vendor/magento/module-cron/Console/Command/CronCommand.php b/vendor/magento/module-cron/Console/Command/CronCommand.php +index 6a9686c514e..4df6888f461 100644 +--- a/vendor/magento/module-cron/Console/Command/CronCommand.php ++++ b/vendor/magento/module-cron/Console/Command/CronCommand.php +@@ -9,10 +9,12 @@ use Symfony\Component\Console\Command\Command; + use Symfony\Component\Console\Input\InputInterface; + use Symfony\Component\Console\Output\OutputInterface; + use Symfony\Component\Console\Input\InputOption; ++use Magento\Framework\App\ObjectManager; + use Magento\Framework\App\ObjectManagerFactory; + use Magento\Store\Model\Store; + use Magento\Store\Model\StoreManager; + use Magento\Cron\Observer\ProcessCronQueueObserver; ++use Magento\Framework\App\DeploymentConfig; + use Magento\Framework\Console\Cli; + use Magento\Framework\Shell\ComplexParameter; + +@@ -34,13 +36,24 @@ class CronCommand extends Command + private $objectManagerFactory; + + /** +- * Constructor ++ * Application deployment configuration + * ++ * @var DeploymentConfig ++ */ ++ private $deploymentConfig; ++ ++ /** + * @param ObjectManagerFactory $objectManagerFactory ++ * @param DeploymentConfig $deploymentConfig Application deployment configuration + */ +- public function __construct(ObjectManagerFactory $objectManagerFactory) +- { ++ public function __construct( ++ ObjectManagerFactory $objectManagerFactory, ++ DeploymentConfig $deploymentConfig = null ++ ) { + $this->objectManagerFactory = $objectManagerFactory; ++ $this->deploymentConfig = $deploymentConfig ?: ObjectManager::getInstance()->get( ++ DeploymentConfig::class ++ ); + parent::__construct(); + } + +@@ -70,10 +83,16 @@ class CronCommand extends Command + } + + /** ++ * Runs cron jobs if cron is not disabled in Magento configurations ++ * + * {@inheritdoc} + */ + protected function execute(InputInterface $input, OutputInterface $output) + { ++ if (!$this->deploymentConfig->get('cron/enabled', 1)) { ++ $output->writeln('' . 'Cron is disabled. Jobs were not run.' . ''); ++ return; ++ } + $omParams = $_SERVER; + $omParams[StoreManager::PARAM_RUN_CODE] = 'admin'; + $omParams[Store::CUSTOM_ENTRY_POINT_PARAM] = true; diff --git a/patches/MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.0.patch b/patches/MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.0.patch new file mode 100644 index 0000000..4931909 --- /dev/null +++ b/patches/MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.0.patch @@ -0,0 +1,39 @@ +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php ++++ b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php +@@ -119,7 +119,7 @@ class PidConsumerManager + */ + public function getPid($consumerName) + { +- $pidFile = $consumerName . static::PID_FILE_EXT; ++ $pidFile = $this->getPidFileName($consumerName); + /** @var WriteInterface $directory */ + $directory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); + +@@ -138,7 +138,7 @@ class PidConsumerManager + */ + public function getPidFilePath($consumerName) + { +- return $this->directoryList->getPath(DirectoryList::VAR_DIR) . '/' . $consumerName . static::PID_FILE_EXT; ++ return $this->directoryList->getPath(DirectoryList::VAR_DIR) . '/' . $this->getPidFileName($consumerName); + } + + /** +@@ -152,4 +152,17 @@ class PidConsumerManager + $file->write(function_exists('posix_getpid') ? posix_getpid() : getmypid()); + $file->close(); + } ++ ++ /** ++ * Returns default file name with PID by consumers name ++ * ++ * @param string $consumerName The consumers name ++ * @return string The file name with PID ++ */ ++ private function getPidFileName($consumerName) ++ { ++ $sanitizedHostname = preg_replace('/[^a-z0-9]/i', '', gethostname()); ++ ++ return $consumerName . '-' . $sanitizedHostname . static::PID_FILE_EXT; ++ } + } diff --git a/patches/MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.4.patch b/patches/MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.4.patch new file mode 100644 index 0000000..67d350f --- /dev/null +++ b/patches/MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.4.patch @@ -0,0 +1,13 @@ +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php ++++ b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +@@ -139,6 +139,8 @@ class ConsumersRunner + */ + private function getPidFilePath($consumerName) + { +- return $consumerName . static::PID_FILE_EXT; ++ $sanitizedHostname = preg_replace('/[^a-z0-9]/i', '', gethostname()); ++ ++ return $consumerName . '-' . $sanitizedHostname . static::PID_FILE_EXT; + } + } diff --git a/patches/MAGECLOUD-2509__remove_permission_check_for_console_application__2.2.0.patch b/patches/MAGECLOUD-2509__remove_permission_check_for_console_application__2.2.0.patch new file mode 100644 index 0000000..54c6aef --- /dev/null +++ b/patches/MAGECLOUD-2509__remove_permission_check_for_console_application__2.2.0.patch @@ -0,0 +1,61 @@ +diff -Naur a/vendor/magento/framework/Console/Cli.php b/vendor/magento/framework/Console/Cli.php +--- a/vendor/magento/framework/Console/Cli.php ++++ b/vendor/magento/framework/Console/Cli.php +@@ -9,7 +9,6 @@ use Magento\Framework\App\Bootstrap; + use Magento\Framework\App\DeploymentConfig; + use Magento\Framework\App\Filesystem\DirectoryList; + use Magento\Framework\App\ProductMetadata; +-use Magento\Framework\App\State; + use Magento\Framework\Composer\ComposerJsonFinder; + use Magento\Framework\Console\Exception\GenerationDirectoryAccessException; + use Magento\Framework\Filesystem\Driver\File; +@@ -19,7 +18,6 @@ use Magento\Setup\Application; + use Magento\Setup\Console\CompilerPreparation; + use Magento\Setup\Model\ObjectManagerProvider; + use Symfony\Component\Console; +-use Zend\ServiceManager\ServiceManager; + + /** + * Magento 2 CLI Application. +@@ -74,7 +72,6 @@ class Cli extends Console\Application + + $this->assertCompilerPreparation(); + $this->initObjectManager(); +- $this->assertGenerationPermissions(); + } catch (\Exception $exception) { + $output = new \Symfony\Component\Console\Output\ConsoleOutput(); + $output->writeln( +@@ -166,33 +163,6 @@ class Cli extends Console\Application + $omProvider->setObjectManager($this->objectManager); + } + +- /** +- * Checks whether generation directory is read-only. +- * Depends on the current mode: +- * production - application will proceed +- * default - application will be terminated +- * developer - application will be terminated +- * +- * @return void +- * @throws GenerationDirectoryAccessException If generation directory is read-only in developer mode +- */ +- private function assertGenerationPermissions() +- { +- /** @var GenerationDirectoryAccess $generationDirectoryAccess */ +- $generationDirectoryAccess = $this->objectManager->create( +- GenerationDirectoryAccess::class, +- ['serviceManager' => $this->serviceManager] +- ); +- /** @var State $state */ +- $state = $this->objectManager->get(State::class); +- +- if ($state->getMode() !== State::MODE_PRODUCTION +- && !$generationDirectoryAccess->check() +- ) { +- throw new GenerationDirectoryAccessException(); +- } +- } +- + /** + * Checks whether compiler is being prepared. + * diff --git a/patches/MAGECLOUD-2509__remove_permission_check_for_console_application__2.2.6.patch b/patches/MAGECLOUD-2509__remove_permission_check_for_console_application__2.2.6.patch new file mode 100644 index 0000000..8c176af --- /dev/null +++ b/patches/MAGECLOUD-2509__remove_permission_check_for_console_application__2.2.6.patch @@ -0,0 +1,53 @@ +diff -Naur a/vendor/magento/framework/Console/Cli.php b/vendor/magento/framework/Console/Cli.php +--- a/vendor/magento/framework/Console/Cli.php ++++ b/vendor/magento/framework/Console/Cli.php +@@ -9,7 +9,6 @@ use Magento\Framework\App\Bootstrap; + use Magento\Framework\App\DeploymentConfig; + use Magento\Framework\App\Filesystem\DirectoryList; + use Magento\Framework\App\ProductMetadata; +-use Magento\Framework\App\State; + use Magento\Framework\Composer\ComposerJsonFinder; + use Magento\Framework\Console\Exception\GenerationDirectoryAccessException; + use Magento\Framework\Filesystem\Driver\File; +@@ -74,7 +73,6 @@ class Cli extends Console\Application + + $this->assertCompilerPreparation(); + $this->initObjectManager(); +- $this->assertGenerationPermissions(); + } catch (\Exception $exception) { + $output = new \Symfony\Component\Console\Output\ConsoleOutput(); + $output->writeln( +@@ -167,33 +165,6 @@ class Cli extends Console\Application + $omProvider->setObjectManager($this->objectManager); + } + +- /** +- * Checks whether generation directory is read-only. +- * Depends on the current mode: +- * production - application will proceed +- * default - application will be terminated +- * developer - application will be terminated +- * +- * @return void +- * @throws GenerationDirectoryAccessException If generation directory is read-only in developer mode +- */ +- private function assertGenerationPermissions() +- { +- /** @var GenerationDirectoryAccess $generationDirectoryAccess */ +- $generationDirectoryAccess = $this->objectManager->create( +- GenerationDirectoryAccess::class, +- ['serviceManager' => $this->serviceManager] +- ); +- /** @var State $state */ +- $state = $this->objectManager->get(State::class); +- +- if ($state->getMode() !== State::MODE_PRODUCTION +- && !$generationDirectoryAccess->check() +- ) { +- throw new GenerationDirectoryAccessException(); +- } +- } +- + /** + * Checks whether compiler is being prepared. + * diff --git a/patches/MAGECLOUD-2521__zendframework1_use_TLS_1.2.patch b/patches/MAGECLOUD-2521__zendframework1_use_TLS_1.2.patch new file mode 100644 index 0000000..a901d45 --- /dev/null +++ b/patches/MAGECLOUD-2521__zendframework1_use_TLS_1.2.patch @@ -0,0 +1,58 @@ +diff -Nuar a/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Imap.php b/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Imap.php +--- a/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Imap.php ++++ b/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Imap.php +@@ -111,7 +111,8 @@ class Zend_Mail_Protocol_Imap + + if ($ssl === 'TLS') { + $result = $this->requestAndResponse('STARTTLS'); +- $result = $result && stream_socket_enable_crypto($this->_socket, true, STREAM_CRYPTO_METHOD_TLS_CLIENT); ++ // TODO: Add STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT in the future when it is supported by PHP ++ $result = $result && stream_socket_enable_crypto($this->_socket, true, STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT); + if (!$result) { + /** + * @see Zend_Mail_Protocol_Exception + +diff -Nuar a/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Pop3.php b/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Pop3.php +--- a/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Pop3.php ++++ b/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Pop3.php +@@ -122,7 +122,8 @@ class Zend_Mail_Protocol_Pop3 + + if ($ssl === 'TLS') { + $this->request('STLS'); +- $result = stream_socket_enable_crypto($this->_socket, true, STREAM_CRYPTO_METHOD_TLS_CLIENT); ++ // TODO: Add STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT in the future when it is supported by PHP ++ $result = stream_socket_enable_crypto($this->_socket, true, STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT); + if (!$result) { + /** + * @see Zend_Mail_Protocol_Exception + +diff -Nuar a/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Smtp.php b/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Smtp.php +--- a/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Smtp.php ++++ b/vendor/magento/zendframework1/library/Zend/Mail/Protocol/Smtp.php +@@ -203,7 +203,8 @@ class Zend_Mail_Protocol_Smtp extends Ze + if ($this->_secure == 'tls') { + $this->_send('STARTTLS'); + $this->_expect(220, 180); +- if (!stream_socket_enable_crypto($this->_socket, true, STREAM_CRYPTO_METHOD_TLS_CLIENT)) { ++ // TODO: Add STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT in the future when it is supported by PHP ++ if (!stream_socket_enable_crypto($this->_socket, true, STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT)) { + /** + * @see Zend_Mail_Protocol_Exception + */ + +diff -Nuar a/vendor/magento/zendframework1/library/Zend/Http/Client/Adapter/Proxy.php b/vendor/magento/zendframework1/library/Zend/Http/Client/Adapter/Proxy.php +--- a/vendor/magento/zendframework1/library/Zend/Http/Client/Adapter/Proxy.php ++++ b/vendor/magento/zendframework1/library/Zend/Http/Client/Adapter/Proxy.php +@@ -297,10 +297,8 @@ class Zend_Http_Client_Adapter_Proxy ext + // If all is good, switch socket to secure mode. We have to fall back + // through the different modes + $modes = array( +- STREAM_CRYPTO_METHOD_TLS_CLIENT, +- STREAM_CRYPTO_METHOD_SSLv3_CLIENT, +- STREAM_CRYPTO_METHOD_SSLv23_CLIENT, +- STREAM_CRYPTO_METHOD_SSLv2_CLIENT ++ // TODO: Add STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT in the future when it is supported by PHP ++ STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT + ); + + $success = false; diff --git a/patches/MAGECLOUD-2573__installation_without_admin_creation__2.1.4.patch b/patches/MAGECLOUD-2573__installation_without_admin_creation__2.1.4.patch new file mode 100644 index 0000000..abb12a7 --- /dev/null +++ b/patches/MAGECLOUD-2573__installation_without_admin_creation__2.1.4.patch @@ -0,0 +1,152 @@ +diff -Naur a/setup/src/Magento/Setup/Model/Installer.php b/setup/src/Magento/Setup/Model/Installer.php +--- a/setup/src/Magento/Setup/Model/Installer.php ++++ b/setup/src/Magento/Setup/Model/Installer.php +@@ -316,7 +316,9 @@ class Installer + [$request[InstallCommand::INPUT_KEY_SALES_ORDER_INCREMENT_PREFIX]], + ]; + } +- $script[] = ['Installing admin user...', 'installAdminUser', [$request]]; ++ if ($this->isAdminDataSet($request)) { ++ $script[] = ['Installing admin user...', 'installAdminUser', [$request]]; ++ } + $script[] = ['Caches clearing:', 'cleanCaches', []]; + $script[] = ['Disabling Maintenance Mode:', 'setMaintenanceMode', [0]]; + $script[] = ['Post installation file permissions check...', 'checkApplicationFilePermissions', []]; +@@ -1318,4 +1320,27 @@ class Installer + $this->log->log($message); + } + } ++ ++ /** ++ * Checks that admin data is not empty in request array ++ * ++ * @param \ArrayObject|array $request ++ * @return bool ++ */ ++ private function isAdminDataSet($request) ++ { ++ $adminData = array_filter($request, function ($value, $key) { ++ return in_array( ++ $key, ++ [ ++ AdminAccount::KEY_EMAIL, ++ AdminAccount::KEY_FIRST_NAME, ++ AdminAccount::KEY_LAST_NAME, ++ AdminAccount::KEY_USER, ++ AdminAccount::KEY_PASSWORD, ++ ] ++ ) && $value !== null; ++ }, ARRAY_FILTER_USE_BOTH); ++ return !empty($adminData); ++ } + } + +diff -Naur a/setup/src/Magento/Setup/Console/Command/InstallCommand.php b/setup/src/Magento/Setup/Console/Command/InstallCommand.php +--- a/setup/src/Magento/Setup/Console/Command/InstallCommand.php ++++ b/setup/src/Magento/Setup/Console/Command/InstallCommand.php +@@ -13,6 +13,7 @@ use Magento\Setup\Model\InstallerFactory; + use Magento\Framework\Setup\ConsoleLogger; + use Symfony\Component\Console\Input\InputOption; + use Magento\Setup\Model\ConfigModel; ++use Magento\Setup\Model\AdminAccount; + + /** + * Command to install Magento application +@@ -90,7 +91,7 @@ class InstallCommand extends AbstractSetupCommand + { + $inputOptions = $this->configModel->getAvailableOptions(); + $inputOptions = array_merge($inputOptions, $this->userConfig->getOptionsList()); +- $inputOptions = array_merge($inputOptions, $this->adminUser->getOptionsList()); ++ $inputOptions = array_merge($inputOptions, $this->adminUser->getOptionsList(InputOption::VALUE_OPTIONAL)); + $inputOptions = array_merge($inputOptions, [ + new InputOption( + self::INPUT_KEY_CLEANUP_DB, +@@ -146,7 +147,7 @@ class InstallCommand extends AbstractSetupCommand + } + } + $errors = $this->configModel->validate($configOptionsToValidate); +- $errors = array_merge($errors, $this->adminUser->validate($input)); ++ $errors = array_merge($errors, $this->validateAdmin($input)); + $errors = array_merge($errors, $this->validate($input)); + $errors = array_merge($errors, $this->userConfig->validate($input)); + +@@ -177,4 +178,23 @@ class InstallCommand extends AbstractSetupCommand + } + return $errors; + } ++ ++ /** ++ * Performs validation of admin options if at least one of them was set. ++ * ++ * @param InputInterface $input ++ * @return array ++ */ ++ private function validateAdmin(InputInterface $input): array ++ { ++ if ($input->getOption(AdminAccount::KEY_FIRST_NAME) ++ || $input->getOption(AdminAccount::KEY_LAST_NAME) ++ || $input->getOption(AdminAccount::KEY_EMAIL) ++ || $input->getOption(AdminAccount::KEY_USER) ++ || $input->getOption(AdminAccount::KEY_PASSWORD) ++ ) { ++ return $this->adminUser->validate($input); ++ } ++ return []; ++ } + } + +diff -Naur a/setup/src/Magento/Setup/Console/Command/AdminUserCreateCommand.php b/setup/src/Magento/Setup/Console/Command/AdminUserCreateCommand.php +--- a/setup/src/Magento/Setup/Console/Command/AdminUserCreateCommand.php ++++ b/setup/src/Magento/Setup/Console/Command/AdminUserCreateCommand.php +@@ -71,25 +71,43 @@ class AdminUserCreateCommand extends AbstractSetupCommand + /** + * Get list of arguments for the command + * ++ * @param int $mode The mode of options. + * @return InputOption[] + */ +- public function getOptionsList() ++ public function getOptionsList($mode = InputOption::VALUE_REQUIRED) + { ++ $requiredStr = ($mode === InputOption::VALUE_REQUIRED ? '(Required) ' : ''); ++ + return [ +- new InputOption(AdminAccount::KEY_USER, null, InputOption::VALUE_REQUIRED, '(Required) Admin user'), +- new InputOption(AdminAccount::KEY_PASSWORD, null, InputOption::VALUE_REQUIRED, '(Required) Admin password'), +- new InputOption(AdminAccount::KEY_EMAIL, null, InputOption::VALUE_REQUIRED, '(Required) Admin email'), ++ new InputOption( ++ AdminAccount::KEY_USER, ++ null, ++ $mode, ++ $requiredStr . 'Admin user' ++ ), ++ new InputOption( ++ AdminAccount::KEY_PASSWORD, ++ null, ++ $mode, ++ $requiredStr . 'Admin password' ++ ), ++ new InputOption( ++ AdminAccount::KEY_EMAIL, ++ null, ++ $mode, ++ $requiredStr . 'Admin email' ++ ), + new InputOption( + AdminAccount::KEY_FIRST_NAME, + null, +- InputOption::VALUE_REQUIRED, +- '(Required) Admin first name' ++ $mode, ++ $requiredStr . 'Admin first name' + ), + new InputOption( + AdminAccount::KEY_LAST_NAME, + null, +- InputOption::VALUE_REQUIRED, +- '(Required) Admin last name' ++ $mode, ++ $requiredStr . 'Admin last name' + ), + ]; + } diff --git a/patches/MAGECLOUD-2573__installation_without_admin_creation__2.2.2.patch b/patches/MAGECLOUD-2573__installation_without_admin_creation__2.2.2.patch new file mode 100644 index 0000000..e88214b --- /dev/null +++ b/patches/MAGECLOUD-2573__installation_without_admin_creation__2.2.2.patch @@ -0,0 +1,198 @@ +diff -Nuar a/setup/src/Magento/Setup/Model/Installer.php b/setup/src/Magento/Setup/Model/Installer.php +--- a/setup/src/Magento/Setup/Model/Installer.php ++++ b/setup/src/Magento/Setup/Model/Installer.php +@@ -316,7 +316,9 @@ class Installer + [$request[InstallCommand::INPUT_KEY_SALES_ORDER_INCREMENT_PREFIX]], + ]; + } +- $script[] = ['Installing admin user...', 'installAdminUser', [$request]]; ++ if ($this->isAdminDataSet($request)) { ++ $script[] = ['Installing admin user...', 'installAdminUser', [$request]]; ++ } + $script[] = ['Caches clearing:', 'cleanCaches', []]; + $script[] = ['Disabling Maintenance Mode:', 'setMaintenanceMode', [0]]; + $script[] = ['Post installation file permissions check...', 'checkApplicationFilePermissions', []]; +@@ -1318,4 +1320,27 @@ class Installer + $this->log->log($message); + } + } ++ ++ /** ++ * Checks that admin data is not empty in request array ++ * ++ * @param \ArrayObject|array $request ++ * @return bool ++ */ ++ private function isAdminDataSet($request) ++ { ++ $adminData = array_filter($request, function ($value, $key) { ++ return in_array( ++ $key, ++ [ ++ AdminAccount::KEY_EMAIL, ++ AdminAccount::KEY_FIRST_NAME, ++ AdminAccount::KEY_LAST_NAME, ++ AdminAccount::KEY_USER, ++ AdminAccount::KEY_PASSWORD, ++ ] ++ ) && $value !== null; ++ }, ARRAY_FILTER_USE_BOTH); ++ return !empty($adminData); ++ } + } + +diff -Nuar a/setup/src/Magento/Setup/Console/Command/InstallCommand.php b/setup/src/Magento/Setup/Console/Command/InstallCommand.php +--- a/setup/src/Magento/Setup/Console/Command/InstallCommand.php ++++ b/setup/src/Magento/Setup/Console/Command/InstallCommand.php +@@ -11,6 +11,7 @@ use Symfony\Component\Console\Input\InputInterface; + use Symfony\Component\Console\Output\OutputInterface; + use Magento\Setup\Model\InstallerFactory; + use Magento\Framework\Setup\ConsoleLogger; ++use Magento\Setup\Model\AdminAccount; + use Symfony\Component\Console\Input\InputOption; + use Magento\Setup\Model\ConfigModel; + use Symfony\Component\Console\Question\Question; +@@ -103,7 +104,7 @@ class InstallCommand extends AbstractSetupCommand + { + $inputOptions = $this->configModel->getAvailableOptions(); + $inputOptions = array_merge($inputOptions, $this->userConfig->getOptionsList()); +- $inputOptions = array_merge($inputOptions, $this->adminUser->getOptionsList()); ++ $inputOptions = array_merge($inputOptions, $this->adminUser->getOptionsList(InputOption::VALUE_OPTIONAL)); + $inputOptions = array_merge($inputOptions, [ + new InputOption( + self::INPUT_KEY_CLEANUP_DB, +@@ -178,7 +179,7 @@ class InstallCommand extends AbstractSetupCommand + } + + $errors = $this->configModel->validate($configOptionsToValidate); +- $errors = array_merge($errors, $this->adminUser->validate($input)); ++ $errors = array_merge($errors, $this->validateAdmin($input)); + $errors = array_merge($errors, $this->validate($input)); + $errors = array_merge($errors, $this->userConfig->validate($input)); + +@@ -247,7 +248,7 @@ class InstallCommand extends AbstractSetupCommand + + $output->writeln(""); + +- foreach ($this->adminUser->getOptionsList() as $option) { ++ foreach ($this->adminUser->getOptionsList(InputOption::VALUE_OPTIONAL) as $option) { + $configOptionsToValidate[$option->getName()] = $this->askQuestion( + $input, + $output, +@@ -338,4 +339,23 @@ class InstallCommand extends AbstractSetupCommand + + return $value; + } ++ ++ /** ++ * Performs validation of admin options if at least one of them was set. ++ * ++ * @param InputInterface $input ++ * @return array ++ */ ++ private function validateAdmin(InputInterface $input): array ++ { ++ if ($input->getOption(AdminAccount::KEY_FIRST_NAME) ++ || $input->getOption(AdminAccount::KEY_LAST_NAME) ++ || $input->getOption(AdminAccount::KEY_EMAIL) ++ || $input->getOption(AdminAccount::KEY_USER) ++ || $input->getOption(AdminAccount::KEY_PASSWORD) ++ ) { ++ return $this->adminUser->validate($input); ++ } ++ return []; ++ } + } + +diff --git a/setup/src/Magento/Setup/Console/Command/AdminUserCreateCommand.php b/setup/src/Magento/Setup/Console/Command/AdminUserCreateCommand.php +--- a/setup/src/Magento/Setup/Console/Command/AdminUserCreateCommand.php ++++ b/setup/src/Magento/Setup/Console/Command/AdminUserCreateCommand.php +@@ -15,6 +15,9 @@ use Symfony\Component\Console\Input\InputOption; + use Symfony\Component\Console\Output\OutputInterface; + use Symfony\Component\Console\Question\Question; + ++/** ++ * Command to create an admin user. ++ */ + class AdminUserCreateCommand extends AbstractSetupCommand + { + /** +@@ -52,6 +55,8 @@ class AdminUserCreateCommand extends AbstractSetupCommand + } + + /** ++ * Creation admin user in interaction mode. ++ * + * @param \Symfony\Component\Console\Input\InputInterface $input + * @param \Symfony\Component\Console\Output\OutputInterface $output + * +@@ -129,6 +134,8 @@ class AdminUserCreateCommand extends AbstractSetupCommand + } + + /** ++ * Add not empty validator. ++ * + * @param \Symfony\Component\Console\Question\Question $question + * @return void + */ +@@ -144,7 +151,7 @@ class AdminUserCreateCommand extends AbstractSetupCommand + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function execute(InputInterface $input, OutputInterface $output) + { +@@ -165,25 +172,43 @@ class AdminUserCreateCommand extends AbstractSetupCommand + /** + * Get list of arguments for the command + * ++ * @param int $mode The mode of options. + * @return InputOption[] + */ +- public function getOptionsList() ++ public function getOptionsList($mode = InputOption::VALUE_REQUIRED) + { ++ $requiredStr = ($mode === InputOption::VALUE_REQUIRED ? '(Required) ' : ''); ++ + return [ +- new InputOption(AdminAccount::KEY_USER, null, InputOption::VALUE_REQUIRED, '(Required) Admin user'), +- new InputOption(AdminAccount::KEY_PASSWORD, null, InputOption::VALUE_REQUIRED, '(Required) Admin password'), +- new InputOption(AdminAccount::KEY_EMAIL, null, InputOption::VALUE_REQUIRED, '(Required) Admin email'), ++ new InputOption( ++ AdminAccount::KEY_USER, ++ null, ++ $mode, ++ $requiredStr . 'Admin user' ++ ), ++ new InputOption( ++ AdminAccount::KEY_PASSWORD, ++ null, ++ $mode, ++ $requiredStr . 'Admin password' ++ ), ++ new InputOption( ++ AdminAccount::KEY_EMAIL, ++ null, ++ $mode, ++ $requiredStr . 'Admin email' ++ ), + new InputOption( + AdminAccount::KEY_FIRST_NAME, + null, +- InputOption::VALUE_REQUIRED, +- '(Required) Admin first name' ++ $mode, ++ $requiredStr . 'Admin first name' + ), + new InputOption( + AdminAccount::KEY_LAST_NAME, + null, +- InputOption::VALUE_REQUIRED, +- '(Required) Admin last name' ++ $mode, ++ $requiredStr . 'Admin last name' + ), + ]; + } diff --git a/patches/MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.13.patch b/patches/MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.13.patch new file mode 100644 index 0000000..ebaf1f8 --- /dev/null +++ b/patches/MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.13.patch @@ -0,0 +1,122 @@ +diff -Nuar a/vendor/magento/module-catalog/Block/Product/View/Options/Type/Date.php b/vendor/magento/module-catalog/Block/Product/View/Options/Type/Date.php +--- a/vendor/magento/module-catalog/Block/Product/View/Options/Type/Date.php ++++ b/vendor/magento/module-catalog/Block/Product/View/Options/Type/Date.php +@@ -83,7 +83,7 @@ class Date extends \Magento\Catalog\Block\Product\View\Options\AbstractOptions + $yearEnd = $this->_catalogProductOptionTypeDate->getYearEnd(); + + $dateFormat = $this->_localeDate->getDateFormat(\IntlDateFormatter::SHORT); +- /** Escape invisible characters which are present in some locales and may corrupt formatting */ ++ /** Escape RTL characters which are present in some locales and corrupt formatting */ + $escapedDateFormat = preg_replace('/[^MmDdYy\/\.\-]/', '', $dateFormat); + $calendar = $this->getLayout()->createBlock( + 'Magento\Framework\View\Element\Html\Date' +diff -Nuar a/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php b/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php +--- a/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php ++++ b/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php +@@ -142,7 +142,7 @@ class Date extends \Magento\Catalog\Model\Product\Option\Type\DefaultType + + if ($this->_dateExists()) { + if ($this->useCalendar()) { +- $timestamp += $this->_localeDate->date($value['date'], null, true)->getTimestamp(); ++ $timestamp += $this->_localeDate->date($value['date'], null, true, false)->getTimestamp(); + } else { + $timestamp += mktime(0, 0, 0, $value['month'], $value['day'], $value['year']); + } +diff -Nuar a/vendor/magento/framework/Stdlib/DateTime/Timezone.php b/vendor/magento/framework/Stdlib/DateTime/Timezone.php +--- a/vendor/magento/framework/Stdlib/DateTime/Timezone.php ++++ b/vendor/magento/framework/Stdlib/DateTime/Timezone.php +@@ -151,27 +151,33 @@ class Timezone implements TimezoneInterface + + /** + * {@inheritdoc} +- * @SuppressWarnings(PHPMD.NPathComplexity) + */ +- public function date($date = null, $locale = null, $useTimezone = true) ++ public function date($date = null, $locale = null, $useTimezone = true, $includeTime = true) + { + $locale = $locale ?: $this->_localeResolver->getLocale(); + $timezone = $useTimezone + ? $this->getConfigTimezone() + : date_default_timezone_get(); + +- if (empty($date)) { +- return new \DateTime('now', new \DateTimeZone($timezone)); +- } elseif ($date instanceof \DateTime) { +- return $date->setTimezone(new \DateTimeZone($timezone)); +- } elseif (!is_numeric($date)) { +- $formatter = new \IntlDateFormatter( +- $locale, +- \IntlDateFormatter::SHORT, +- \IntlDateFormatter::NONE +- ); +- $date = $formatter->parse($date) ?: (new \DateTime($date))->getTimestamp(); ++ switch (true) { ++ case (empty($date)): ++ return new \DateTime('now', new \DateTimeZone($timezone)); ++ case ($date instanceof \DateTime): ++ return $date->setTimezone(new \DateTimeZone($timezone)); ++ case ($date instanceof \DateTimeImmutable): ++ return new \DateTime($date->format('Y-m-d H:i:s'), $date->getTimezone()); ++ case (!is_numeric($date)): ++ $timeType = $includeTime ? \IntlDateFormatter::SHORT : \IntlDateFormatter::NONE; ++ $formatter = new \IntlDateFormatter( ++ $locale, ++ \IntlDateFormatter::SHORT, ++ $timeType, ++ new \DateTimeZone($timezone) ++ ); ++ $date = $formatter->parse($date) ?: (new \DateTime($date))->getTimestamp(); ++ break; + } ++ + return (new \DateTime(null, new \DateTimeZone($timezone)))->setTimestamp($date); + } + +@@ -195,7 +201,7 @@ class Timezone implements TimezoneInterface + { + $formatTime = $showTime ? $format : \IntlDateFormatter::NONE; + +- if (!($date instanceof \DateTime)) { ++ if (!($date instanceof \DateTimeInterface)) { + $date = new \DateTime($date); + } + +@@ -258,7 +264,7 @@ class Timezone implements TimezoneInterface + $timezone = null, + $pattern = null + ) { +- if (!($date instanceof \DateTime)) { ++ if (!($date instanceof \DateTimeInterface)) { + $date = new \DateTime($date); + } + +@@ -294,8 +300,12 @@ class Timezone implements TimezoneInterface + */ + public function convertConfigTimeToUtc($date, $format = 'Y-m-d H:i:s') + { +- if (!($date instanceof \DateTime)) { +- $date = new \DateTime($date, new \DateTimeZone($this->getConfigTimezone())); ++ if (!($date instanceof \DateTimeInterface)) { ++ if ($date instanceof \DateTimeImmutable) { ++ $date = new \DateTime($date->format('Y-m-d H:i:s'), new \DateTimeZone($this->getConfigTimezone())); ++ } else { ++ $date = new \DateTime($date, new \DateTimeZone($this->getConfigTimezone())); ++ } + } else { + if ($date->getTimezone()->getName() !== $this->getConfigTimezone()) { + throw new LocalizedException( +diff -Nuar a/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php b/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php +--- a/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php ++++ b/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php +@@ -65,9 +65,10 @@ interface TimezoneInterface + * @param mixed $date + * @param string $locale + * @param bool $useTimezone ++ * @param bool $includeTime + * @return \DateTime + */ +- public function date($date = null, $locale = null, $useTimezone = true); ++ public function date($date = null, $locale = null, $useTimezone = true, $includeTime = true); + + /** + * Create \DateTime object with date converted to scope timezone and scope Locale diff --git a/patches/MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.4.patch b/patches/MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.4.patch new file mode 100644 index 0000000..f2f668b --- /dev/null +++ b/patches/MAGECLOUD-2602__fix_timezone_parsing_for_cron__2.1.4.patch @@ -0,0 +1,170 @@ +diff -Naur a/vendor/magento/module-catalog/Block/Product/View/Options/Type/Date.php b/vendor/magento/module-catalog/Block/Product/View/Options/Type/Date.php +--- a/vendor/magento/module-catalog/Block/Product/View/Options/Type/Date.php ++++ b/vendor/magento/module-catalog/Block/Product/View/Options/Type/Date.php +@@ -1,6 +1,6 @@ + _catalogProductOptionTypeDate->getYearStart(); + $yearEnd = $this->_catalogProductOptionTypeDate->getYearEnd(); + ++ $dateFormat = $this->_localeDate->getDateFormat(\IntlDateFormatter::SHORT); ++ /** Escape RTL characters which are present in some locales and corrupt formatting */ ++ $escapedDateFormat = preg_replace('/[^MmDdYy\/\.\-]/', '', $dateFormat); + $calendar = $this->getLayout()->createBlock( + 'Magento\Framework\View\Element\Html\Date' + )->setId( +@@ -93,7 +96,7 @@ class Date extends \Magento\Catalog\Block\Product\View\Options\AbstractOptions + )->setImage( + $this->getViewFileUrl('Magento_Theme::calendar.png') + )->setDateFormat( +- $this->_localeDate->getDateFormat(\IntlDateFormatter::SHORT) ++ $escapedDateFormat + )->setValue( + $value + )->setYearsRange( +diff -Naur a/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php b/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php +--- a/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php ++++ b/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php +@@ -1,6 +1,6 @@ + _dateExists()) { + if ($this->useCalendar()) { +- $timestamp += (new \DateTime($value['date']))->getTimestamp(); ++ $timestamp += $this->_localeDate->date($value['date'], null, true, false)->getTimestamp(); + } else { + $timestamp += mktime(0, 0, 0, $value['month'], $value['day'], $value['year']); + } +diff -Naur a/vendor/magento/framework/Stdlib/DateTime/Timezone.php b/vendor/magento/framework/Stdlib/DateTime/Timezone.php +--- a/vendor/magento/framework/Stdlib/DateTime/Timezone.php ++++ b/vendor/magento/framework/Stdlib/DateTime/Timezone.php +@@ -1,6 +1,6 @@ + _localeResolver->getLocale(); + $timezone = $useTimezone + ? $this->getConfigTimezone() + : date_default_timezone_get(); + +- if (empty($date)) { +- return new \DateTime('now', new \DateTimeZone($timezone)); +- } elseif ($date instanceof \DateTime) { +- return $date->setTimezone(new \DateTimeZone($timezone)); +- } elseif (!is_numeric($date)) { +- $formatter = new \IntlDateFormatter( +- $locale, +- \IntlDateFormatter::SHORT, +- \IntlDateFormatter::SHORT, +- new \DateTimeZone($timezone) +- ); +- $date = $formatter->parse($date) ?: (new \DateTime($date))->getTimestamp(); ++ switch (true) { ++ case (empty($date)): ++ return new \DateTime('now', new \DateTimeZone($timezone)); ++ case ($date instanceof \DateTime): ++ return $date->setTimezone(new \DateTimeZone($timezone)); ++ case ($date instanceof \DateTimeImmutable): ++ return new \DateTime($date->format('Y-m-d H:i:s'), $date->getTimezone()); ++ case (!is_numeric($date)): ++ $timeType = $includeTime ? \IntlDateFormatter::SHORT : \IntlDateFormatter::NONE; ++ $formatter = new \IntlDateFormatter( ++ $locale, ++ \IntlDateFormatter::SHORT, ++ $timeType, ++ new \DateTimeZone($timezone) ++ ); ++ $date = $formatter->parse($date) ?: (new \DateTime($date))->getTimestamp(); ++ break; + } ++ + return (new \DateTime(null, new \DateTimeZone($timezone)))->setTimestamp($date); + } + +@@ -196,7 +201,7 @@ class Timezone implements TimezoneInterface + { + $formatTime = $showTime ? $format : \IntlDateFormatter::NONE; + +- if (!($date instanceof \DateTime)) { ++ if (!($date instanceof \DateTimeInterface)) { + $date = new \DateTime($date); + } + +@@ -259,7 +264,7 @@ class Timezone implements TimezoneInterface + $timezone = null, + $pattern = null + ) { +- if (!($date instanceof \DateTime)) { ++ if (!($date instanceof \DateTimeInterface)) { + $date = new \DateTime($date); + } + +@@ -295,8 +300,12 @@ class Timezone implements TimezoneInterface + */ + public function convertConfigTimeToUtc($date, $format = 'Y-m-d H:i:s') + { +- if (!($date instanceof \DateTime)) { +- $date = new \DateTime($date, new \DateTimeZone($this->getConfigTimezone())); ++ if (!($date instanceof \DateTimeInterface)) { ++ if ($date instanceof \DateTimeImmutable) { ++ $date = new \DateTime($date->format('Y-m-d H:i:s'), new \DateTimeZone($this->getConfigTimezone())); ++ } else { ++ $date = new \DateTime($date, new \DateTimeZone($this->getConfigTimezone())); ++ } + } else { + if ($date->getTimezone()->getName() !== $this->getConfigTimezone()) { + throw new LocalizedException( +diff -Nuar a/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php b/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php +--- a/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php ++++ b/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php +@@ -1,6 +1,6 @@ + _catalogProductOptionTypeDate->getYearStart(); + $yearEnd = $this->_catalogProductOptionTypeDate->getYearEnd(); + ++ $dateFormat = $this->_localeDate->getDateFormat(\IntlDateFormatter::SHORT); ++ /** Escape RTL characters which are present in some locales and corrupt formatting */ ++ $escapedDateFormat = preg_replace('/[^MmDdYy\/\.\-]/', '', $dateFormat); + $calendar = $this->getLayout()->createBlock( + 'Magento\Framework\View\Element\Html\Date' + )->setId( +@@ -93,7 +96,7 @@ class Date extends \Magento\Catalog\Block\Product\View\Options\AbstractOptions + )->setImage( + $this->getViewFileUrl('Magento_Theme::calendar.png') + )->setDateFormat( +- $this->_localeDate->getDateFormat(\IntlDateFormatter::SHORT) ++ $escapedDateFormat + )->setValue( + $value + )->setYearsRange( +diff -Nuar a/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php b/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php +--- a/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php ++++ b/vendor/magento/module-catalog/Model/Product/Option/Type/Date.php +@@ -1,6 +1,6 @@ + _dateExists()) { + if ($this->useCalendar()) { +- $timestamp += (new \DateTime($value['date']))->getTimestamp(); ++ $timestamp += $this->_localeDate->date($value['date'], null, true, false)->getTimestamp(); + } else { + $timestamp += mktime(0, 0, 0, $value['month'], $value['day'], $value['year']); + } +diff -Nuar a/vendor/magento/framework/Stdlib/DateTime/Timezone.php b/vendor/magento/framework/Stdlib/DateTime/Timezone.php +--- a/vendor/magento/framework/Stdlib/DateTime/Timezone.php ++++ b/vendor/magento/framework/Stdlib/DateTime/Timezone.php +@@ -1,6 +1,6 @@ + _localeResolver->getLocale(); + $timezone = $useTimezone + ? $this->getConfigTimezone() + : date_default_timezone_get(); + +- if (empty($date)) { +- return new \DateTime('now', new \DateTimeZone($timezone)); +- } elseif ($date instanceof \DateTime) { +- return $date->setTimezone(new \DateTimeZone($timezone)); +- } elseif (!is_numeric($date)) { +- $formatter = new \IntlDateFormatter( +- $locale, +- \IntlDateFormatter::SHORT, +- \IntlDateFormatter::SHORT, +- new \DateTimeZone($timezone) +- ); +- $date = $formatter->parse($date) ?: (new \DateTime($date))->getTimestamp(); ++ switch (true) { ++ case (empty($date)): ++ return new \DateTime('now', new \DateTimeZone($timezone)); ++ case ($date instanceof \DateTime): ++ return $date->setTimezone(new \DateTimeZone($timezone)); ++ case ($date instanceof \DateTimeImmutable): ++ return new \DateTime($date->format('Y-m-d H:i:s'), $date->getTimezone()); ++ case (!is_numeric($date)): ++ $timeType = $includeTime ? \IntlDateFormatter::SHORT : \IntlDateFormatter::NONE; ++ $formatter = new \IntlDateFormatter( ++ $locale, ++ \IntlDateFormatter::SHORT, ++ $timeType, ++ new \DateTimeZone($timezone) ++ ); ++ $date = $formatter->parse($date) ?: (new \DateTime($date))->getTimestamp(); ++ break; + } ++ + return (new \DateTime(null, new \DateTimeZone($timezone)))->setTimestamp($date); + } + +@@ -196,7 +201,7 @@ class Timezone implements TimezoneInterface + { + $formatTime = $showTime ? $format : \IntlDateFormatter::NONE; + +- if (!($date instanceof \DateTime)) { ++ if (!($date instanceof \DateTimeInterface)) { + $date = new \DateTime($date); + } + +@@ -259,7 +264,7 @@ class Timezone implements TimezoneInterface + $timezone = null, + $pattern = null + ) { +- if (!($date instanceof \DateTime)) { ++ if (!($date instanceof \DateTimeInterface)) { + $date = new \DateTime($date); + } + +@@ -295,8 +300,12 @@ class Timezone implements TimezoneInterface + */ + public function convertConfigTimeToUtc($date, $format = 'Y-m-d H:i:s') + { +- if (!($date instanceof \DateTime)) { +- $date = new \DateTime($date, new \DateTimeZone($this->getConfigTimezone())); ++ if (!($date instanceof \DateTimeInterface)) { ++ if ($date instanceof \DateTimeImmutable) { ++ $date = new \DateTime($date->format('Y-m-d H:i:s'), new \DateTimeZone($this->getConfigTimezone())); ++ } else { ++ $date = new \DateTime($date, new \DateTimeZone($this->getConfigTimezone())); ++ } + } else { + if ($date->getTimezone()->getName() !== $this->getConfigTimezone()) { + throw new LocalizedException( +diff -Nuar a/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php b/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php +--- a/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php ++++ b/vendor/magento/framework/Stdlib/DateTime/TimezoneInterface.php +@@ -1,6 +1,6 @@ + useAttachment = $useAttachment; + $this->useShortAttachment = $useShortAttachment; + $this->includeContextAndExtra = $includeContextAndExtra; +- if ($this->includeContextAndExtra) { ++ ++ if ($this->includeContextAndExtra && $this->useShortAttachment) { + $this->lineFormatter = new LineFormatter; + } + } +@@ -139,35 +141,26 @@ class SlackHandler extends SocketHandler + 'channel' => $this->channel, + 'username' => $this->username, + 'text' => '', +- 'attachments' => array() ++ 'attachments' => array(), + ); + + if ($this->useAttachment) { + $attachment = array( + 'fallback' => $record['message'], +- 'color' => $this->getAttachmentColor($record['level']) ++ 'color' => $this->getAttachmentColor($record['level']), ++ 'fields' => array(), + ); + + if ($this->useShortAttachment) { +- $attachment['fields'] = array( +- array( +- 'title' => $record['level_name'], +- 'value' => $record['message'], +- 'short' => false +- ) +- ); ++ $attachment['title'] = $record['level_name']; ++ $attachment['text'] = $record['message']; + } else { +- $attachment['fields'] = array( +- array( +- 'title' => 'Message', +- 'value' => $record['message'], +- 'short' => false +- ), +- array( +- 'title' => 'Level', +- 'value' => $record['level_name'], +- 'short' => true +- ) ++ $attachment['title'] = 'Message'; ++ $attachment['text'] = $record['message']; ++ $attachment['fields'][] = array( ++ 'title' => 'Level', ++ 'value' => $record['level_name'], ++ 'short' => true, + ); + } + +@@ -177,7 +170,7 @@ class SlackHandler extends SocketHandler + $attachment['fields'][] = array( + 'title' => "Extra", + 'value' => $this->stringify($record['extra']), +- 'short' => $this->useShortAttachment ++ 'short' => $this->useShortAttachment, + ); + } else { + // Add all extra fields as individual fields in attachment +@@ -185,7 +178,7 @@ class SlackHandler extends SocketHandler + $attachment['fields'][] = array( + 'title' => $var, + 'value' => $val, +- 'short' => $this->useShortAttachment ++ 'short' => $this->useShortAttachment, + ); + } + } +@@ -196,7 +189,7 @@ class SlackHandler extends SocketHandler + $attachment['fields'][] = array( + 'title' => "Context", + 'value' => $this->stringify($record['context']), +- 'short' => $this->useShortAttachment ++ 'short' => $this->useShortAttachment, + ); + } else { + // Add all context fields as individual fields in attachment +@@ -204,7 +197,7 @@ class SlackHandler extends SocketHandler + $attachment['fields'][] = array( + 'title' => $var, + 'value' => $val, +- 'short' => $this->useShortAttachment ++ 'short' => $this->useShortAttachment, + ); + } + } +@@ -248,6 +241,10 @@ class SlackHandler extends SocketHandler + protected function write(array $record) + { + parent::write($record); ++ $res = $this->getResource(); ++ if (is_resource($res)) { ++ @fread($res, 2048); ++ } + $this->closeSocket(); + } + +@@ -275,8 +272,7 @@ class SlackHandler extends SocketHandler + /** + * Stringifies an array of key/value pairs to be used in attachment fields + * +- * @param array $fields +- * @access protected ++ * @param array $fields + * @return string + */ + protected function stringify($fields) +diff -Nuar a/vendor/monolog/monolog/src/Monolog/Handler/SocketHandler.php b/vendor/monolog/monolog/src/Monolog/Handler/SocketHandler.php +index a3e7252e..e4c3c37f 100644 +--- a/vendor/monolog/monolog/src/Monolog/Handler/SocketHandler.php ++++ b/vendor/monolog/monolog/src/Monolog/Handler/SocketHandler.php +@@ -41,6 +41,15 @@ class SocketHandler extends AbstractProcessingHandler + $this->connectionTimeout = (float) ini_get('default_socket_timeout'); + } + ++ /** ++ * @return resource|null ++ */ ++ protected function getResource() ++ { ++ return $this->resource; ++ } ++ ++ + /** + * Connect (if necessary) and write to the socket + * diff --git a/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.13.patch b/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.13.patch new file mode 100644 index 0000000..9e9b478 --- /dev/null +++ b/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.13.patch @@ -0,0 +1,76 @@ +diff -Nuar a/vendor/magento/framework/DB/Statement/Pdo/Mysql.php b/vendor/magento/framework/DB/Statement/Pdo/Mysql.php +--- a/vendor/magento/framework/DB/Statement/Pdo/Mysql.php ++++ b/vendor/magento/framework/DB/Statement/Pdo/Mysql.php +@@ -3,23 +3,20 @@ + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ ++namespace Magento\Framework\DB\Statement\Pdo; + +-// @codingStandardsIgnoreFile ++use Magento\Framework\DB\Statement\Parameter; + + /** + * Mysql DB Statement + * + * @author Magento Core Team + */ +-namespace Magento\Framework\DB\Statement\Pdo; +- +-use Magento\Framework\DB\Statement\Parameter; +- + class Mysql extends \Zend_Db_Statement_Pdo + { ++ + /** +- * Executes statement with binding values to it. +- * Allows transferring specific options to DB driver. ++ * Executes statement with binding values to it. Allows transferring specific options to DB driver. + * + * @param array $params Array of values to bind to parameter placeholders. + * @return bool +@@ -63,11 +60,9 @@ class Mysql extends \Zend_Db_Statement_Pdo + $statement->bindParam($paramName, $bindValues[$name], $dataType, $length, $driverOptions); + } + +- try { ++ return $this->tryExecute(function () use ($statement) { + return $statement->execute(); +- } catch (\PDOException $e) { +- throw new \Zend_Db_Statement_Exception($e->getMessage(), (int)$e->getCode(), $e); +- } ++ }); + } + + /** +@@ -92,7 +87,29 @@ class Mysql extends \Zend_Db_Statement_Pdo + if ($specialExecute) { + return $this->_executeWithBinding($params); + } else { +- return parent::_execute($params); ++ return $this->tryExecute(function () use ($params) { ++ return $params !== null ? $this->_stmt->execute($params) : $this->_stmt->execute(); ++ }); ++ } ++ } ++ ++ /** ++ * Executes query and avoid warnings. ++ * ++ * @param callable $callback ++ * @return bool ++ * @throws \Zend_Db_Statement_Exception ++ */ ++ private function tryExecute($callback) ++ { ++ $previousLevel = error_reporting(\E_ERROR); // disable warnings for PDO bugs #63812, #74401 ++ try { ++ return $callback(); ++ } catch (\PDOException $e) { ++ $message = sprintf('%s, query was: %s', $e->getMessage(), $this->_stmt->queryString); ++ throw new \Zend_Db_Statement_Exception($message, (int)$e->getCode(), $e); ++ } finally { ++ error_reporting($previousLevel); + } + } + } diff --git a/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.4.patch b/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.4.patch new file mode 100644 index 0000000..b2b704d --- /dev/null +++ b/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.4.patch @@ -0,0 +1,79 @@ +diff -Nuar a/vendor/magento/framework/DB/Statement/Pdo/Mysql.php b/vendor/magento/framework/DB/Statement/Pdo/Mysql.php +--- a/vendor/magento/framework/DB/Statement/Pdo/Mysql.php ++++ b/vendor/magento/framework/DB/Statement/Pdo/Mysql.php +@@ -1,25 +1,22 @@ + + */ +-namespace Magento\Framework\DB\Statement\Pdo; +- +-use Magento\Framework\DB\Statement\Parameter; +- + class Mysql extends \Zend_Db_Statement_Pdo + { ++ + /** +- * Executes statement with binding values to it. +- * Allows transferring specific options to DB driver. ++ * Executes statement with binding values to it. Allows transferring specific options to DB driver. + * + * @param array $params Array of values to bind to parameter placeholders. + * @return bool +@@ -63,11 +60,9 @@ class Mysql extends \Zend_Db_Statement_Pdo + $statement->bindParam($paramName, $bindValues[$name], $dataType, $length, $driverOptions); + } + +- try { ++ return $this->tryExecute(function () use ($statement) { + return $statement->execute(); +- } catch (\PDOException $e) { +- throw new \Zend_Db_Statement_Exception($e->getMessage(), (int)$e->getCode(), $e); +- } ++ }); + } + + /** +@@ -92,7 +87,29 @@ class Mysql extends \Zend_Db_Statement_Pdo + if ($specialExecute) { + return $this->_executeWithBinding($params); + } else { +- return parent::_execute($params); ++ return $this->tryExecute(function () use ($params) { ++ return $params !== null ? $this->_stmt->execute($params) : $this->_stmt->execute(); ++ }); ++ } ++ } ++ ++ /** ++ * Executes query and avoid warnings. ++ * ++ * @param callable $callback ++ * @return bool ++ * @throws \Zend_Db_Statement_Exception ++ */ ++ private function tryExecute($callback) ++ { ++ $previousLevel = error_reporting(\E_ERROR); // disable warnings for PDO bugs #63812, #74401 ++ try { ++ return $callback(); ++ } catch (\PDOException $e) { ++ $message = sprintf('%s, query was: %s', $e->getMessage(), $this->_stmt->queryString); ++ throw new \Zend_Db_Statement_Exception($message, (int)$e->getCode(), $e); ++ } finally { ++ error_reporting($previousLevel); + } + } + } diff --git a/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.5.patch b/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.5.patch new file mode 100644 index 0000000..641edce --- /dev/null +++ b/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.1.5.patch @@ -0,0 +1,79 @@ +diff -Nuar a/vendor/magento/framework/DB/Statement/Pdo/Mysql.php b/vendor/magento/framework/DB/Statement/Pdo/Mysql.php +--- a/vendor/magento/framework/DB/Statement/Pdo/Mysql.php ++++ b/vendor/magento/framework/DB/Statement/Pdo/Mysql.php +@@ -1,25 +1,22 @@ + + */ +-namespace Magento\Framework\DB\Statement\Pdo; +- +-use Magento\Framework\DB\Statement\Parameter; +- + class Mysql extends \Zend_Db_Statement_Pdo + { ++ + /** +- * Executes statement with binding values to it. +- * Allows transferring specific options to DB driver. ++ * Executes statement with binding values to it. Allows transferring specific options to DB driver. + * + * @param array $params Array of values to bind to parameter placeholders. + * @return bool +@@ -63,11 +60,9 @@ class Mysql extends \Zend_Db_Statement_Pdo + $statement->bindParam($paramName, $bindValues[$name], $dataType, $length, $driverOptions); + } + +- try { ++ return $this->tryExecute(function () use ($statement) { + return $statement->execute(); +- } catch (\PDOException $e) { +- throw new \Zend_Db_Statement_Exception($e->getMessage(), (int)$e->getCode(), $e); +- } ++ }); + } + + /** +@@ -92,7 +87,29 @@ class Mysql extends \Zend_Db_Statement_Pdo + if ($specialExecute) { + return $this->_executeWithBinding($params); + } else { +- return parent::_execute($params); ++ return $this->tryExecute(function () use ($params) { ++ return $params !== null ? $this->_stmt->execute($params) : $this->_stmt->execute(); ++ }); ++ } ++ } ++ ++ /** ++ * Executes query and avoid warnings. ++ * ++ * @param callable $callback ++ * @return bool ++ * @throws \Zend_Db_Statement_Exception ++ */ ++ private function tryExecute($callback) ++ { ++ $previousLevel = error_reporting(\E_ERROR); // disable warnings for PDO bugs #63812, #74401 ++ try { ++ return $callback(); ++ } catch (\PDOException $e) { ++ $message = sprintf('%s, query was: %s', $e->getMessage(), $this->_stmt->queryString); ++ throw new \Zend_Db_Statement_Exception($message, (int)$e->getCode(), $e); ++ } finally { ++ error_reporting($previousLevel); + } + } + } diff --git a/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.2.0.patch b/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.2.0.patch new file mode 100644 index 0000000..d6ed435 --- /dev/null +++ b/patches/MAGECLOUD-2820__implement_isolated_connections_mechanism__2.2.0.patch @@ -0,0 +1,75 @@ +diff -Nuar a/vendor/magento/framework/DB/Statement/Pdo/Mysql.php b/vendor/magento/framework/DB/Statement/Pdo/Mysql.php +--- a/vendor/magento/framework/DB/Statement/Pdo/Mysql.php ++++ b/vendor/magento/framework/DB/Statement/Pdo/Mysql.php +@@ -3,21 +3,20 @@ + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ ++namespace Magento\Framework\DB\Statement\Pdo; ++ ++use Magento\Framework\DB\Statement\Parameter; + + /** + * Mysql DB Statement + * + * @author Magento Core Team + */ +-namespace Magento\Framework\DB\Statement\Pdo; +- +-use Magento\Framework\DB\Statement\Parameter; +- + class Mysql extends \Zend_Db_Statement_Pdo + { ++ + /** +- * Executes statement with binding values to it. +- * Allows transferring specific options to DB driver. ++ * Executes statement with binding values to it. Allows transferring specific options to DB driver. + * + * @param array $params Array of values to bind to parameter placeholders. + * @return bool +@@ -61,11 +60,9 @@ class Mysql extends \Zend_Db_Statement_Pdo + $statement->bindParam($paramName, $bindValues[$name], $dataType, $length, $driverOptions); + } + +- try { ++ return $this->tryExecute(function () use ($statement) { + return $statement->execute(); +- } catch (\PDOException $e) { +- throw new \Zend_Db_Statement_Exception($e->getMessage(), (int)$e->getCode(), $e); +- } ++ }); + } + + /** +@@ -90,7 +87,29 @@ class Mysql extends \Zend_Db_Statement_Pdo + if ($specialExecute) { + return $this->_executeWithBinding($params); + } else { +- return parent::_execute($params); ++ return $this->tryExecute(function () use ($params) { ++ return $params !== null ? $this->_stmt->execute($params) : $this->_stmt->execute(); ++ }); ++ } ++ } ++ ++ /** ++ * Executes query and avoid warnings. ++ * ++ * @param callable $callback ++ * @return bool ++ * @throws \Zend_Db_Statement_Exception ++ */ ++ private function tryExecute($callback) ++ { ++ $previousLevel = error_reporting(\E_ERROR); // disable warnings for PDO bugs #63812, #74401 ++ try { ++ return $callback(); ++ } catch (\PDOException $e) { ++ $message = sprintf('%s, query was: %s', $e->getMessage(), $this->_stmt->queryString); ++ throw new \Zend_Db_Statement_Exception($message, (int)$e->getCode(), $e); ++ } finally { ++ error_reporting($previousLevel); + } + } + } diff --git a/patches/MAGECLOUD-2822__configure_max_execution_time.patch b/patches/MAGECLOUD-2822__configure_max_execution_time.patch new file mode 100644 index 0000000..1fd810c --- /dev/null +++ b/patches/MAGECLOUD-2822__configure_max_execution_time.patch @@ -0,0 +1,107 @@ +diff -Nuar a/vendor/magento/module-deploy/Console/DeployStaticOptions.php b/vendor/magento/module-deploy/Console/DeployStaticOptions.php +--- a/vendor/magento/module-deploy/Console/DeployStaticOptions.php ++++ b/vendor/magento/module-deploy/Console/DeployStaticOptions.php +@@ -6,6 +6,7 @@ + + namespace Magento\Deploy\Console; + ++use Magento\Deploy\Process\Queue; + use Symfony\Component\Console\Input\InputOption; + use Symfony\Component\Console\Input\InputArgument; + +@@ -57,6 +58,11 @@ class DeployStaticOptions + */ + const JOBS_AMOUNT = 'jobs'; + ++ /** ++ * Key for max execution time option ++ */ ++ const MAX_EXECUTION_TIME = 'max-execution-time'; ++ + /** + * Force run of static deploy + */ +@@ -150,6 +156,7 @@ public function getOptionsList() + * Basic options + * + * @return array ++ * @SuppressWarnings(PHPMD.ExcessiveMethodLength) + */ + private function getBasicOptions() + { +@@ -216,6 +223,13 @@ private function getBasicOptions() + 'Enable parallel processing using the specified number of jobs.', + self::DEFAULT_JOBS_AMOUNT + ), ++ new InputOption( ++ self::MAX_EXECUTION_TIME, ++ null, ++ InputOption::VALUE_OPTIONAL, ++ 'The maximum expected execution time of deployment static process (in seconds).', ++ Queue::DEFAULT_MAX_EXEC_TIME ++ ), + new InputOption( + self::SYMLINK_LOCALE, + null, +diff -Nuar a/vendor/magento/module-deploy/Service/DeployStaticContent.php b/vendor/magento/module-deploy/Service/DeployStaticContent.php +--- a/vendor/magento/module-deploy/Service/DeployStaticContent.php ++++ b/vendor/magento/module-deploy/Service/DeployStaticContent.php +@@ -85,24 +85,26 @@ public function deploy(array $options) + return; + } + +- $queue = $this->queueFactory->create( +- [ +- 'logger' => $this->logger, +- 'options' => $options, +- 'maxProcesses' => $this->getProcessesAmount($options), +- 'deployPackageService' => $this->objectManager->create( +- \Magento\Deploy\Service\DeployPackage::class, +- [ +- 'logger' => $this->logger +- ] +- ) +- ] +- ); ++ $queueOptions = [ ++ 'logger' => $this->logger, ++ 'options' => $options, ++ 'maxProcesses' => $this->getProcessesAmount($options), ++ 'deployPackageService' => $this->objectManager->create( ++ \Magento\Deploy\Service\DeployPackage::class, ++ [ ++ 'logger' => $this->logger ++ ] ++ ) ++ ]; ++ ++ if (isset($options[Options::MAX_EXECUTION_TIME])) { ++ $queueOptions['maxExecTime'] = (int)$options[Options::MAX_EXECUTION_TIME]; ++ } + + $deployStrategy = $this->deployStrategyFactory->create( + $options[Options::STRATEGY], + [ +- 'queue' => $queue ++ 'queue' => $this->queueFactory->create($queueOptions) + ] + ); + +@@ -133,6 +135,8 @@ public function deploy(array $options) + } + + /** ++ * Returns amount of parallel processes, returns zero if option wasn't set. ++ * + * @param array $options + * @return int + */ +@@ -142,6 +146,8 @@ private function getProcessesAmount(array $options) + } + + /** ++ * Checks if need to refresh only version. ++ * + * @param array $options + * @return bool + */ diff --git a/patches/MAGECLOUD-2822__configure_max_execution_time_2.3.1.patch b/patches/MAGECLOUD-2822__configure_max_execution_time_2.3.1.patch new file mode 100644 index 0000000..9f98713 --- /dev/null +++ b/patches/MAGECLOUD-2822__configure_max_execution_time_2.3.1.patch @@ -0,0 +1,89 @@ +diff -Nuar a/vendor/magento/module-deploy/Console/DeployStaticOptions.php b/vendor/magento/module-deploy/Console/DeployStaticOptions.php +--- a/vendor/magento/module-deploy/Console/DeployStaticOptions.php ++++ b/vendor/magento/module-deploy/Console/DeployStaticOptions.php +@@ -6,6 +6,7 @@ + + namespace Magento\Deploy\Console; + ++use Magento\Deploy\Process\Queue; + use Symfony\Component\Console\Input\InputOption; + use Symfony\Component\Console\Input\InputArgument; + +@@ -57,6 +58,11 @@ class DeployStaticOptions + */ + const JOBS_AMOUNT = 'jobs'; + ++ /** ++ * Key for max execution time option ++ */ ++ const MAX_EXECUTION_TIME = 'max-execution-time'; ++ + /** + * Force run of static deploy + */ +@@ -150,6 +156,7 @@ public function getOptionsList() + * Basic options + * + * @return array ++ * @SuppressWarnings(PHPMD.ExcessiveMethodLength) + */ + private function getBasicOptions() + { +@@ -216,6 +223,13 @@ private function getBasicOptions() + 'Enable parallel processing using the specified number of jobs.', + self::DEFAULT_JOBS_AMOUNT + ), ++ new InputOption( ++ self::MAX_EXECUTION_TIME, ++ null, ++ InputOption::VALUE_OPTIONAL, ++ 'The maximum expected execution time of deployment static process (in seconds).', ++ Queue::DEFAULT_MAX_EXEC_TIME ++ ), + new InputOption( + self::SYMLINK_LOCALE, + null, +diff -Nuar a/vendor/magento/module-deploy/Service/DeployStaticContent.php b/vendor/magento/module-deploy/Service/DeployStaticContent.php +--- a/vendor/magento/module-deploy/Service/DeployStaticContent.php ++++ b/vendor/magento/module-deploy/Service/DeployStaticContent.php +@@ -88,24 +88,26 @@ class DeployStaticContent + return; + } + +- $queue = $this->queueFactory->create( +- [ +- 'logger' => $this->logger, +- 'options' => $options, +- 'maxProcesses' => $this->getProcessesAmount($options), +- 'deployPackageService' => $this->objectManager->create( +- \Magento\Deploy\Service\DeployPackage::class, +- [ +- 'logger' => $this->logger +- ] +- ) +- ] +- ); ++ $queueOptions = [ ++ 'logger' => $this->logger, ++ 'options' => $options, ++ 'maxProcesses' => $this->getProcessesAmount($options), ++ 'deployPackageService' => $this->objectManager->create( ++ \Magento\Deploy\Service\DeployPackage::class, ++ [ ++ 'logger' => $this->logger ++ ] ++ ) ++ ]; ++ ++ if (isset($options[Options::MAX_EXECUTION_TIME])) { ++ $queueOptions['maxExecTime'] = (int)$options[Options::MAX_EXECUTION_TIME]; ++ } + + $deployStrategy = $this->deployStrategyFactory->create( + $options[Options::STRATEGY], + [ +- 'queue' => $queue ++ 'queue' => $this->queueFactory->create($queueOptions) + ] + ); + diff --git a/patches/MAGECLOUD-2850_fix_amazon_payment_module__2.2.6.patch b/patches/MAGECLOUD-2850_fix_amazon_payment_module__2.2.6.patch new file mode 100644 index 0000000..df9b29e --- /dev/null +++ b/patches/MAGECLOUD-2850_fix_amazon_payment_module__2.2.6.patch @@ -0,0 +1,177 @@ +diff -Nuar a/vendor/amzn/amazon-pay-module/etc/di.xml b/vendor/amzn/amazon-pay-module/etc/di.xml +index c954f48..e585eae 100644 +--- a/vendor/amzn/amazon-pay-module/etc/di.xml ++++ b/vendor/amzn/amazon-pay-module/etc/di.xml +@@ -39,24 +39,20 @@ + + + +- ++ + + amazon_error_mapping.xml + + +- ++ + + Amazon\Payment\Gateway\ErrorMapper\VirtualConfigReader + amazon_error_mapper + + +- ++ + +- Amazon\Payment\Gateway\ErrorMapper\VirtualMappingData +- ++ Amazon\Payment\Gateway\ErrorMapper\VirtualMappingData + + + +@@ -120,15 +116,12 @@ + + + +- Amazon\Payment\Gateway\Request\AuthorizationRequest +- ++ Amazon\Payment\Gateway\Request\AuthorizationRequest + Amazon\Payment\Gateway\Response\CompleteAuthHandler + Amazon\Payment\Gateway\Http\TransferFactory + AmazonAuthorizationValidators + Amazon\Payment\Gateway\Http\Client\AuthorizeClient +- Amazon\Payment\Gateway\ErrorMapper\VirtualErrorMessageMapper +- ++ Amazon\Payment\Gateway\ErrorMapper\VirtualErrorMessageMapper + + + +@@ -141,30 +134,24 @@ + + + +- Amazon\Payment\Gateway\Request\AuthorizationRequest +- ++ Amazon\Payment\Gateway\Request\AuthorizationRequest + Amazon\Payment\Gateway\Response\CompleteSaleHandler + Amazon\Payment\Gateway\Http\TransferFactory + AmazonAuthorizationValidators + Amazon\Payment\Gateway\Http\Client\CaptureClient +- Amazon\Payment\Gateway\ErrorMapper\VirtualErrorMessageMapper +- ++ Amazon\Payment\Gateway\ErrorMapper\VirtualErrorMessageMapper + + + + + + +- Amazon\Payment\Gateway\Request\SettlementRequest +- ++ Amazon\Payment\Gateway\Request\SettlementRequest + Amazon\Payment\Gateway\Response\SettlementHandler + Amazon\Payment\Gateway\Http\TransferFactory + AmazonAuthorizationValidators + Amazon\Payment\Gateway\Http\Client\SettlementClient +- Amazon\Payment\Gateway\ErrorMapper\VirtualErrorMessageMapper +- ++ Amazon\Payment\Gateway\ErrorMapper\VirtualErrorMessageMapper + + + +@@ -183,12 +170,9 @@ + Amazon\Payment\Gateway\Request\RefundRequest + Amazon\Payment\Gateway\Response\RefundHandler + Amazon\Payment\Gateway\Http\TransferFactory +- Amazon\Payment\Gateway\Validator\AuthorizationValidator +- ++ Amazon\Payment\Gateway\Validator\AuthorizationValidator + Amazon\Payment\Gateway\Http\Client\RefundClient +- Amazon\Payment\Gateway\ErrorMapper\VirtualErrorMessageMapper +- ++ Amazon\Payment\Gateway\ErrorMapper\VirtualErrorMessageMapper + + + +@@ -198,12 +182,9 @@ + Amazon\Payment\Gateway\Request\VoidRequest + Amazon\Payment\Gateway\Response\VoidHandler + Amazon\Payment\Gateway\Http\TransferFactory +- Amazon\Payment\Gateway\Validator\AuthorizationValidator +- ++ Amazon\Payment\Gateway\Validator\AuthorizationValidator + Amazon\Payment\Gateway\Http\Client\VoidClient +- Amazon\Payment\Gateway\ErrorMapper\VirtualErrorMessageMapper +- ++ Amazon\Payment\Gateway\ErrorMapper\VirtualErrorMessageMapper + + + +@@ -237,26 +218,22 @@ + + + +- ++ + + + + + +- ++ + + + +- ++ + + + + +- ++ + + + +@@ -280,17 +257,14 @@ + + + +- Magento\Framework\Notification\NotifierInterface\Proxy +- ++ Magento\Framework\Notification\NotifierInterface\Proxy + + + + + + Amazon\Payment\Model\Ipn\CaptureProcessor\Proxy +- Amazon\Payment\Model\Ipn\AuthorizationProcessor\Proxy +- ++ Amazon\Payment\Model\Ipn\AuthorizationProcessor\Proxy + Amazon\Payment\Model\Ipn\OrderProcessor\Proxy + Amazon\Payment\Model\Ipn\RefundProcessor\Proxy + +@@ -310,8 +284,7 @@ + + + +- ++ + + + diff --git a/patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.1.16.patch b/patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.1.16.patch new file mode 100644 index 0000000..7d4df14 --- /dev/null +++ b/patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.1.16.patch @@ -0,0 +1,40 @@ +diff -Nuar a/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php b/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php +--- a/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php ++++ b/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php +@@ -111,6 +111,13 @@ class Cm_Cache_Backend_Redis extends Zend_Cache_Backend implements Zend_Cache_Ba + */ + protected $_luaMaxCStack = 5000; + ++ /** ++ * If 'retry_reads_on_master' is truthy then reads will be retried against master when slave returns "(nil)" value ++ * ++ * @var boolean ++ */ ++ protected $_retryReadsOnMaster = false; ++ + /** + * @var stdClass + */ +@@ -316,6 +323,10 @@ class Cm_Cache_Backend_Redis extends Zend_Cache_Backend implements Zend_Cache_Ba + $this->_luaMaxCStack = (int) $options['lua_max_c_stack']; + } + ++ if (isset($options['retry_reads_on_master'])) { ++ $this->_retryReadsOnMaster = (bool) $options['retry_reads_on_master']; ++ } ++ + if (isset($options['auto_expire_lifetime'])) { + $this->_autoExpireLifetime = (int) $options['auto_expire_lifetime']; + } +@@ -371,6 +382,11 @@ class Cm_Cache_Backend_Redis extends Zend_Cache_Backend implements Zend_Cache_Ba + { + if ($this->_slave) { + $data = $this->_slave->hGet(self::PREFIX_KEY.$id, self::FIELD_DATA); ++ ++ // Prevent compounded effect of cache flood on asynchronously replicating master/slave setup ++ if ($this->_retryReadsOnMaster && $data === false) { ++ $data = $this->_redis->hGet(self::PREFIX_KEY.$id, self::FIELD_DATA); ++ } + } else { + $data = $this->_redis->hGet(self::PREFIX_KEY.$id, self::FIELD_DATA); + } diff --git a/patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.2.3.patch b/patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.2.3.patch new file mode 100644 index 0000000..c64650e --- /dev/null +++ b/patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.2.3.patch @@ -0,0 +1,40 @@ +diff -Nuar a/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php b/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php +--- a/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php ++++ b/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php +@@ -111,6 +111,13 @@ class Cm_Cache_Backend_Redis extends Zend_Cache_Backend implements Zend_Cache_Ba + */ + protected $_luaMaxCStack = 5000; + ++ /** ++ * If 'retry_reads_on_master' is truthy then reads will be retried against master when slave returns "(nil)" value ++ * ++ * @var boolean ++ */ ++ protected $_retryReadsOnMaster = false; ++ + /** + * @var stdClass + */ +@@ -326,6 +333,10 @@ class Cm_Cache_Backend_Redis extends Zend_Cache_Backend implements Zend_Cache_Ba + $this->_luaMaxCStack = (int) $options['lua_max_c_stack']; + } + ++ if (isset($options['retry_reads_on_master'])) { ++ $this->_retryReadsOnMaster = (bool) $options['retry_reads_on_master']; ++ } ++ + if (isset($options['auto_expire_lifetime'])) { + $this->_autoExpireLifetime = (int) $options['auto_expire_lifetime']; + } +@@ -428,6 +439,11 @@ class Cm_Cache_Backend_Redis extends Zend_Cache_Backend implements Zend_Cache_Ba + { + if ($this->_slave) { + $data = $this->_slave->hGet(self::PREFIX_KEY.$id, self::FIELD_DATA); ++ ++ // Prevent compounded effect of cache flood on asynchronously replicating master/slave setup ++ if ($this->_retryReadsOnMaster && $data === false) { ++ $data = $this->_redis->hGet(self::PREFIX_KEY.$id, self::FIELD_DATA); ++ } + } else { + $data = $this->_redis->hGet(self::PREFIX_KEY.$id, self::FIELD_DATA); + } diff --git a/patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.3.0.patch b/patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.3.0.patch new file mode 100644 index 0000000..568c2a1 --- /dev/null +++ b/patches/MAGECLOUD-2899__fix_redis_slave_configuration__2.3.0.patch @@ -0,0 +1,40 @@ +diff -Nuar a/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php b/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php +--- a/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php ++++ b/vendor/colinmollenhour/cache-backend-redis/Cm/Cache/Backend/Redis.php +@@ -111,6 +111,13 @@ class Cm_Cache_Backend_Redis extends Zend_Cache_Backend implements Zend_Cache_Ba + */ + protected $_luaMaxCStack = 5000; + ++ /** ++ * If 'retry_reads_on_master' is truthy then reads will be retried against master when slave returns "(nil)" value ++ * ++ * @var boolean ++ */ ++ protected $_retryReadsOnMaster = false; ++ + /** + * @var stdClass + */ +@@ -339,6 +346,10 @@ class Cm_Cache_Backend_Redis extends Zend_Cache_Backend implements Zend_Cache_Ba + $this->_luaMaxCStack = (int) $options['lua_max_c_stack']; + } + ++ if (isset($options['retry_reads_on_master'])) { ++ $this->_retryReadsOnMaster = (bool) $options['retry_reads_on_master']; ++ } ++ + if (isset($options['auto_expire_lifetime'])) { + $this->_autoExpireLifetime = (int) $options['auto_expire_lifetime']; + } +@@ -441,6 +452,11 @@ class Cm_Cache_Backend_Redis extends Zend_Cache_Backend implements Zend_Cache_Ba + { + if ($this->_slave) { + $data = $this->_slave->hGet(self::PREFIX_KEY.$id, self::FIELD_DATA); ++ ++ // Prevent compounded effect of cache flood on asynchronously replicating master/slave setup ++ if ($this->_retryReadsOnMaster && $data === false) { ++ $data = $this->_redis->hGet(self::PREFIX_KEY.$id, self::FIELD_DATA); ++ } + } else { + $data = $this->_redis->hGet(self::PREFIX_KEY.$id, self::FIELD_DATA); + } diff --git a/patches/MAGECLOUD-3054__add_zookeeper_and_flock_locks__2.2.5.patch b/patches/MAGECLOUD-3054__add_zookeeper_and_flock_locks__2.2.5.patch new file mode 100644 index 0000000..8df8c6d --- /dev/null +++ b/patches/MAGECLOUD-3054__add_zookeeper_and_flock_locks__2.2.5.patch @@ -0,0 +1,1068 @@ +diff -Naur a/app/etc/di.xml b/app/etc/di.xml +--- a/app/etc/di.xml ++++ b/app/etc/di.xml +@@ -37,7 +37,7 @@ + + + +- ++ + + + +diff -Naur a/vendor/magento/framework/Lock/Backend/FileLock.php b/vendor/magento/framework/Lock/Backend/FileLock.php +--- /dev/null ++++ b/vendor/magento/framework/Lock/Backend/FileLock.php +@@ -0,0 +1,194 @@ ++fileDriver = $fileDriver; ++ $this->path = rtrim($path, '/') . '/'; ++ ++ try { ++ if (!$this->fileDriver->isExists($this->path)) { ++ $this->fileDriver->createDirectory($this->path); ++ } ++ } catch (FileSystemException $exception) { ++ throw new RuntimeException( ++ new Phrase('Cannot create the directory for locks: %1', [$this->path]), ++ $exception ++ ); ++ } ++ } ++ ++ /** ++ * Acquires a lock by name ++ * ++ * @param string $name The lock name ++ * @param int $timeout Timeout in seconds. A negative timeout value means infinite timeout ++ * @return bool Returns true if the lock is acquired, otherwise returns false ++ * @throws RuntimeException Throws RuntimeException if cannot acquires the lock because FS problems ++ */ ++ public function lock(string $name, int $timeout = -1): bool ++ { ++ try { ++ $lockFile = $this->getLockPath($name); ++ $fileResource = $this->fileDriver->fileOpen($lockFile, 'w+'); ++ $skipDeadline = $timeout < 0; ++ $deadline = microtime(true) + $timeout; ++ ++ while (!$this->tryToLock($fileResource)) { ++ if (!$skipDeadline && $deadline <= microtime(true)) { ++ $this->fileDriver->fileClose($fileResource); ++ return false; ++ } ++ usleep($this->sleepCycle); ++ } ++ } catch (FileSystemException $exception) { ++ throw new RuntimeException(new Phrase('Cannot acquire a lock.'), $exception); ++ } ++ ++ $this->locks[$lockFile] = $fileResource; ++ return true; ++ } ++ ++ /** ++ * Checks if a lock exists by name ++ * ++ * @param string $name The lock name ++ * @return bool Returns true if the lock exists, otherwise returns false ++ * @throws RuntimeException Throws RuntimeException if cannot check that the lock exists ++ */ ++ public function isLocked(string $name): bool ++ { ++ $lockFile = $this->getLockPath($name); ++ $result = false; ++ ++ try { ++ if ($this->fileDriver->isExists($lockFile)) { ++ $fileResource = $this->fileDriver->fileOpen($lockFile, 'w+'); ++ if ($this->tryToLock($fileResource)) { ++ $result = false; ++ } else { ++ $result = true; ++ } ++ $this->fileDriver->fileClose($fileResource); ++ } ++ } catch (FileSystemException $exception) { ++ throw new RuntimeException(new Phrase('Cannot verify that the lock exists.'), $exception); ++ } ++ ++ return $result; ++ } ++ ++ /** ++ * Remove the lock by name ++ * ++ * @param string $name The lock name ++ * @return bool If the lock is removed returns true, otherwise returns false ++ */ ++ public function unlock(string $name): bool ++ { ++ $lockFile = $this->getLockPath($name); ++ ++ if (isset($this->locks[$lockFile]) && $this->tryToUnlock($this->locks[$lockFile])) { ++ unset($this->locks[$lockFile]); ++ return true; ++ } ++ ++ return false; ++ } ++ ++ /** ++ * Returns the full path to the lock file by name ++ * ++ * @param string $name The lock name ++ * @return string The path to the lock file ++ */ ++ private function getLockPath(string $name): string ++ { ++ return $this->path . $name; ++ } ++ ++ /** ++ * Tries to lock a file resource ++ * ++ * @param resource $resource The file resource ++ * @return bool If the lock is acquired returns true, otherwise returns false ++ */ ++ private function tryToLock($resource): bool ++ { ++ try { ++ return $this->fileDriver->fileLock($resource, LOCK_EX | LOCK_NB); ++ } catch (FileSystemException $exception) { ++ return false; ++ } ++ } ++ ++ /** ++ * Tries to unlock a file resource ++ * ++ * @param resource $resource The file resource ++ * @return bool If the lock is removed returns true, otherwise returns false ++ */ ++ private function tryToUnlock($resource): bool ++ { ++ try { ++ return $this->fileDriver->fileLock($resource, LOCK_UN | LOCK_NB); ++ } catch (FileSystemException $exception) { ++ return false; ++ } ++ } ++} +diff -Naur a/vendor/magento/framework/Lock/Backend/Zookeeper.php b/vendor/magento/framework/Lock/Backend/Zookeeper.php +--- /dev/null ++++ b/vendor/magento/framework/Lock/Backend/Zookeeper.php +@@ -0,0 +1,280 @@ ++\Zookeeper::PERM_ALL, 'scheme' => 'world', 'id' => 'anyone']]; ++ ++ /** ++ * The mapping list of the lock name with the full lock path ++ * ++ * @var array ++ */ ++ private $locks = []; ++ ++ /** ++ * The default path to storage locks ++ */ ++ const DEFAULT_PATH = '/magento/locks'; ++ ++ /** ++ * @param string $host The host to connect to Zookeeper ++ * @param string $path The base path to locks in Zookeeper ++ * @throws RuntimeException ++ */ ++ public function __construct(string $host, string $path = self::DEFAULT_PATH) ++ { ++ if (!$path) { ++ throw new RuntimeException( ++ new Phrase('The path needs to be a non-empty string.') ++ ); ++ } ++ ++ if (!$host) { ++ throw new RuntimeException( ++ new Phrase('The host needs to be a non-empty string.') ++ ); ++ } ++ ++ $this->host = $host; ++ $this->path = rtrim($path, '/') . '/'; ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * You can see the lock algorithm by the link ++ * @link https://zookeeper.apache.org/doc/r3.1.2/recipes.html#sc_recipes_Locks ++ * ++ * @throws RuntimeException ++ */ ++ public function lock(string $name, int $timeout = -1): bool ++ { ++ $skipDeadline = $timeout < 0; ++ $lockPath = $this->getFullPathToLock($name); ++ $deadline = microtime(true) + $timeout; ++ ++ if (!$this->checkAndCreateParentNode($lockPath)) { ++ throw new RuntimeException(new Phrase('Failed creating the path %1', [$lockPath])); ++ } ++ ++ $lockKey = $this->getProvider() ++ ->create($lockPath, '1', $this->acl, \Zookeeper::EPHEMERAL | \Zookeeper::SEQUENCE); ++ ++ if (!$lockKey) { ++ throw new RuntimeException(new Phrase('Failed creating lock %1', [$lockPath])); ++ } ++ ++ while ($this->isAnyLock($lockKey, $this->getIndex($lockKey))) { ++ if (!$skipDeadline && $deadline <= microtime(true)) { ++ $this->getProvider()->delete($lockKey); ++ return false; ++ } ++ ++ usleep($this->sleepCycle); ++ } ++ ++ $this->locks[$name] = $lockKey; ++ ++ return true; ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * @throws RuntimeException ++ */ ++ public function unlock(string $name): bool ++ { ++ if (!isset($this->locks[$name])) { ++ return false; ++ } ++ ++ return $this->getProvider()->delete($this->locks[$name]); ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * @throws RuntimeException ++ */ ++ public function isLocked(string $name): bool ++ { ++ return $this->isAnyLock($this->getFullPathToLock($name)); ++ } ++ ++ /** ++ * Gets full path to lock by its name ++ * ++ * @param string $name ++ * @return string ++ */ ++ private function getFullPathToLock(string $name): string ++ { ++ return $this->path . $name . '/' . $this->lockName; ++ } ++ ++ /** ++ * Initiolizes and returns Zookeeper provider ++ * ++ * @return \Zookeeper ++ * @throws RuntimeException ++ */ ++ private function getProvider(): \Zookeeper ++ { ++ if (!$this->zookeeper) { ++ $this->zookeeper = new \Zookeeper($this->host); ++ } ++ ++ $deadline = microtime(true) + $this->connectionTimeout; ++ while ($this->zookeeper->getState() != \Zookeeper::CONNECTED_STATE) { ++ if ($deadline <= microtime(true)) { ++ throw new RuntimeException(new Phrase('Zookeeper connection timed out!')); ++ } ++ usleep($this->sleepCycle); ++ } ++ ++ return $this->zookeeper; ++ } ++ ++ /** ++ * Checks and creates base path recursively ++ * ++ * @param string $path ++ * @return bool ++ * @throws RuntimeException ++ */ ++ private function checkAndCreateParentNode(string $path): bool ++ { ++ $path = dirname($path); ++ if ($this->getProvider()->exists($path)) { ++ return true; ++ } ++ ++ if (!$this->checkAndCreateParentNode($path)) { ++ return false; ++ } ++ ++ if ($this->getProvider()->create($path, '1', $this->acl)) { ++ return true; ++ } ++ ++ return $this->getProvider()->exists($path); ++ } ++ ++ /** ++ * Gets int increment of lock key ++ * ++ * @param string $key ++ * @return int|null ++ */ ++ private function getIndex(string $key) ++ { ++ if (!preg_match('/' . $this->lockName . '([0-9]+)$/', $key, $matches)) { ++ return null; ++ } ++ ++ return intval($matches[1]); ++ } ++ ++ /** ++ * Checks if there is any sequence node under parent of $fullKey. ++ * ++ * At first checks that the $fullKey node is present, if not - returns false. ++ * If $indexKey is non-null and there is a smaller index than $indexKey then returns true, ++ * otherwise returns false. ++ * ++ * @param string $fullKey The full path without any sequence info ++ * @param int|null $indexKey The index to compare ++ * @return bool ++ * @throws RuntimeException ++ */ ++ private function isAnyLock(string $fullKey, int $indexKey = null): bool ++ { ++ $parent = dirname($fullKey); ++ ++ if (!$this->getProvider()->exists($parent)) { ++ return false; ++ } ++ ++ $children = $this->getProvider()->getChildren($parent); ++ ++ if (null === $indexKey && !empty($children)) { ++ return true; ++ } ++ ++ foreach ($children as $childKey) { ++ $childIndex = $this->getIndex($childKey); ++ ++ if (null === $childIndex) { ++ continue; ++ } ++ ++ if ($childIndex < $indexKey) { ++ return true; ++ } ++ } ++ ++ return false; ++ } ++} +diff -Naur a/vendor/magento/framework/Lock/LockBackendFactory.php b/vendor/magento/framework/Lock/LockBackendFactory.php +--- /dev/null ++++ b/vendor/magento/framework/Lock/LockBackendFactory.php +@@ -0,0 +1,111 @@ ++ DatabaseLock::class, ++ self::LOCK_ZOOKEEPER => ZookeeperLock::class, ++ self::LOCK_CACHE => CacheLock::class, ++ self::LOCK_FILE => FileLock::class, ++ ]; ++ ++ /** ++ * @param ObjectManagerInterface $objectManager The Object Manager instance ++ * @param DeploymentConfig $deploymentConfig The Application deployment configuration ++ */ ++ public function __construct( ++ ObjectManagerInterface $objectManager, ++ DeploymentConfig $deploymentConfig ++ ) { ++ $this->objectManager = $objectManager; ++ $this->deploymentConfig = $deploymentConfig; ++ } ++ ++ /** ++ * Creates an instance of LockManagerInterface using information from deployment config ++ * ++ * @return LockManagerInterface ++ * @throws RuntimeException ++ */ ++ public function create(): LockManagerInterface ++ { ++ $provider = $this->deploymentConfig->get('lock/provider', self::LOCK_DB); ++ $config = $this->deploymentConfig->get('lock/config', []); ++ ++ if (!isset($this->lockers[$provider])) { ++ throw new RuntimeException(new Phrase('Unknown locks provider: %1', [$provider])); ++ } ++ ++ if (self::LOCK_ZOOKEEPER === $provider && !extension_loaded(self::LOCK_ZOOKEEPER)) { ++ throw new RuntimeException(new Phrase('php extension Zookeeper is not installed.')); ++ } ++ ++ return $this->objectManager->create($this->lockers[$provider], $config); ++ } ++} +diff -Naur a/vendor/magento/framework/Lock/LockManagerInterface.php b/vendor/magento/framework/Lock/LockManagerInterface.php +--- a/vendor/magento/framework/Lock/LockManagerInterface.php ++++ b/vendor/magento/framework/Lock/LockManagerInterface.php +@@ -3,8 +3,8 @@ + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +- + declare(strict_types=1); ++ + namespace Magento\Framework\Lock; + + /** +diff -Naur a/vendor/magento/framework/Lock/Proxy.php b/vendor/magento/framework/Lock/Proxy.php +--- /dev/null ++++ b/vendor/magento/framework/Lock/Proxy.php +@@ -0,0 +1,83 @@ ++factory = $factory; ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * @throws RuntimeException ++ */ ++ public function isLocked(string $name): bool ++ { ++ return $this->getLocker()->isLocked($name); ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * @throws RuntimeException ++ */ ++ public function lock(string $name, int $timeout = -1): bool ++ { ++ return $this->getLocker()->lock($name, $timeout); ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * @throws RuntimeException ++ */ ++ public function unlock(string $name): bool ++ { ++ return $this->getLocker()->unlock($name); ++ } ++ ++ /** ++ * Gets LockManagerInterface implementation using Factory ++ * ++ * @return LockManagerInterface ++ * @throws RuntimeException ++ */ ++ private function getLocker(): LockManagerInterface ++ { ++ if (!$this->locker) { ++ $this->locker = $this->factory->create(); ++ } ++ ++ return $this->locker; ++ } ++} +diff -Naur a/setup/src/Magento/Setup/Model/ConfigOptionsList.php b/setup/src/Magento/Setup/Model/ConfigOptionsList.php +--- a/setup/src/Magento/Setup/Model/ConfigOptionsList.php ++++ b/setup/src/Magento/Setup/Model/ConfigOptionsList.php +@@ -44,7 +44,8 @@ class ConfigOptionsList implements ConfigOptionsListInterface + private $configOptionsListClasses = [ + \Magento\Setup\Model\ConfigOptionsList\Session::class, + \Magento\Setup\Model\ConfigOptionsList\Cache::class, +- \Magento\Setup\Model\ConfigOptionsList\PageCache::class ++ \Magento\Setup\Model\ConfigOptionsList\PageCache::class, ++ \Magento\Setup\Model\ConfigOptionsList\Lock::class, + ]; + + /** +diff -Naur a/setup/src/Magento/Setup/Model/ConfigOptionsList/Lock.php b/setup/src/Magento/Setup/Model/ConfigOptionsList/Lock.php +--- /dev/null ++++ b/setup/src/Magento/Setup/Model/ConfigOptionsList/Lock.php +@@ -0,0 +1,342 @@ ++ [ ++ self::INPUT_KEY_LOCK_PROVIDER => self::CONFIG_PATH_LOCK_PROVIDER, ++ self::INPUT_KEY_LOCK_DB_PREFIX => self::CONFIG_PATH_LOCK_DB_PREFIX, ++ ], ++ LockBackendFactory::LOCK_ZOOKEEPER => [ ++ self::INPUT_KEY_LOCK_PROVIDER => self::CONFIG_PATH_LOCK_PROVIDER, ++ self::INPUT_KEY_LOCK_ZOOKEEPER_HOST => self::CONFIG_PATH_LOCK_ZOOKEEPER_HOST, ++ self::INPUT_KEY_LOCK_ZOOKEEPER_PATH => self::CONFIG_PATH_LOCK_ZOOKEEPER_PATH, ++ ], ++ LockBackendFactory::LOCK_CACHE => [ ++ self::INPUT_KEY_LOCK_PROVIDER => self::CONFIG_PATH_LOCK_PROVIDER, ++ ], ++ LockBackendFactory::LOCK_FILE => [ ++ self::INPUT_KEY_LOCK_PROVIDER => self::CONFIG_PATH_LOCK_PROVIDER, ++ self::INPUT_KEY_LOCK_FILE_PATH => self::CONFIG_PATH_LOCK_FILE_PATH, ++ ], ++ ]; ++ ++ /** ++ * The list of default values ++ * ++ * @var array ++ */ ++ private $defaultConfigValues = [ ++ self::INPUT_KEY_LOCK_PROVIDER => LockBackendFactory::LOCK_DB, ++ self::INPUT_KEY_LOCK_DB_PREFIX => null, ++ self::INPUT_KEY_LOCK_ZOOKEEPER_PATH => ZookeeperLock::DEFAULT_PATH, ++ ]; ++ ++ /** ++ * @inheritdoc ++ */ ++ public function getOptions() ++ { ++ return [ ++ new SelectConfigOption( ++ self::INPUT_KEY_LOCK_PROVIDER, ++ SelectConfigOption::FRONTEND_WIZARD_SELECT, ++ $this->validLockProviders, ++ self::CONFIG_PATH_LOCK_PROVIDER, ++ 'Lock provider name', ++ LockBackendFactory::LOCK_DB ++ ), ++ new TextConfigOption( ++ self::INPUT_KEY_LOCK_DB_PREFIX, ++ TextConfigOption::FRONTEND_WIZARD_TEXT, ++ self::CONFIG_PATH_LOCK_DB_PREFIX, ++ 'Installation specific lock prefix to avoid lock conflicts' ++ ), ++ new TextConfigOption( ++ self::INPUT_KEY_LOCK_ZOOKEEPER_HOST, ++ TextConfigOption::FRONTEND_WIZARD_TEXT, ++ self::CONFIG_PATH_LOCK_ZOOKEEPER_HOST, ++ 'Host and port to connect to Zookeeper cluster. For example: 127.0.0.1:2181' ++ ), ++ new TextConfigOption( ++ self::INPUT_KEY_LOCK_ZOOKEEPER_PATH, ++ TextConfigOption::FRONTEND_WIZARD_TEXT, ++ self::CONFIG_PATH_LOCK_ZOOKEEPER_PATH, ++ 'The path where Zookeeper will save locks. The default path is: ' . ZookeeperLock::DEFAULT_PATH ++ ), ++ new TextConfigOption( ++ self::INPUT_KEY_LOCK_FILE_PATH, ++ TextConfigOption::FRONTEND_WIZARD_TEXT, ++ self::CONFIG_PATH_LOCK_FILE_PATH, ++ 'The path where file locks will be saved.' ++ ), ++ ]; ++ } ++ ++ /** ++ * @inheritdoc ++ */ ++ public function createConfig(array $options, DeploymentConfig $deploymentConfig) ++ { ++ $configData = new ConfigData(ConfigFilePool::APP_ENV); ++ $configData->setOverrideWhenSave(true); ++ $lockProvider = $this->getLockProvider($options, $deploymentConfig); ++ ++ $this->setDefaultConfiguration($configData, $deploymentConfig, $lockProvider); ++ ++ foreach ($this->mappingInputKeyToConfigPath[$lockProvider] as $input => $path) { ++ if (isset($options[$input])) { ++ $configData->set($path, $options[$input]); ++ } ++ } ++ ++ return $configData; ++ } ++ ++ /** ++ * @inheritdoc ++ */ ++ public function validate(array $options, DeploymentConfig $deploymentConfig) ++ { ++ $lockProvider = $this->getLockProvider($options, $deploymentConfig); ++ switch ($lockProvider) { ++ case LockBackendFactory::LOCK_ZOOKEEPER: ++ $errors = $this->validateZookeeperConfig($options, $deploymentConfig); ++ break; ++ case LockBackendFactory::LOCK_FILE: ++ $errors = $this->validateFileConfig($options, $deploymentConfig); ++ break; ++ case LockBackendFactory::LOCK_CACHE: ++ case LockBackendFactory::LOCK_DB: ++ $errors = []; ++ break; ++ default: ++ $errors[] = 'The lock provider ' . $lockProvider . ' does not exist.'; ++ } ++ ++ return $errors; ++ } ++ ++ /** ++ * Validates File locks configuration ++ * ++ * @param array $options ++ * @param DeploymentConfig $deploymentConfig ++ * @return array ++ */ ++ private function validateFileConfig(array $options, DeploymentConfig $deploymentConfig): array ++ { ++ $errors = []; ++ ++ $path = $options[self::INPUT_KEY_LOCK_FILE_PATH] ++ ?? $deploymentConfig->get( ++ self::CONFIG_PATH_LOCK_FILE_PATH, ++ $this->getDefaultValue(self::INPUT_KEY_LOCK_FILE_PATH) ++ ); ++ ++ if (!$path) { ++ $errors[] = 'The path needs to be a non-empty string.'; ++ } ++ ++ return $errors; ++ } ++ ++ /** ++ * Validates Zookeeper configuration ++ * ++ * @param array $options ++ * @param DeploymentConfig $deploymentConfig ++ * @return array ++ */ ++ private function validateZookeeperConfig(array $options, DeploymentConfig $deploymentConfig): array ++ { ++ $errors = []; ++ ++ if (!extension_loaded(LockBackendFactory::LOCK_ZOOKEEPER)) { ++ $errors[] = 'php extension Zookeeper is not installed.'; ++ } ++ ++ $host = $options[self::INPUT_KEY_LOCK_ZOOKEEPER_HOST] ++ ?? $deploymentConfig->get( ++ self::CONFIG_PATH_LOCK_ZOOKEEPER_HOST, ++ $this->getDefaultValue(self::INPUT_KEY_LOCK_ZOOKEEPER_HOST) ++ ); ++ $path = $options[self::INPUT_KEY_LOCK_ZOOKEEPER_PATH] ++ ?? $deploymentConfig->get( ++ self::CONFIG_PATH_LOCK_ZOOKEEPER_PATH, ++ $this->getDefaultValue(self::INPUT_KEY_LOCK_ZOOKEEPER_PATH) ++ ); ++ ++ if (!$path) { ++ $errors[] = 'Zookeeper path needs to be a non-empty string.'; ++ } ++ ++ if (!$host) { ++ $errors[] = 'Zookeeper host is should be set.'; ++ } ++ ++ return $errors; ++ } ++ ++ /** ++ * Returns the name of lock provider ++ * ++ * @param array $options ++ * @param DeploymentConfig $deploymentConfig ++ * @return string ++ */ ++ private function getLockProvider(array $options, DeploymentConfig $deploymentConfig): string ++ { ++ if (!isset($options[self::INPUT_KEY_LOCK_PROVIDER])) { ++ return (string) $deploymentConfig->get( ++ self::CONFIG_PATH_LOCK_PROVIDER, ++ $this->getDefaultValue(self::INPUT_KEY_LOCK_PROVIDER) ++ ); ++ } ++ ++ return (string) $options[self::INPUT_KEY_LOCK_PROVIDER]; ++ } ++ ++ /** ++ * Sets default configuration for locks ++ * ++ * @param ConfigData $configData ++ * @param DeploymentConfig $deploymentConfig ++ * @param string $lockProvider ++ * @return ConfigData ++ */ ++ private function setDefaultConfiguration( ++ ConfigData $configData, ++ DeploymentConfig $deploymentConfig, ++ string $lockProvider ++ ) { ++ foreach ($this->mappingInputKeyToConfigPath[$lockProvider] as $input => $path) { ++ $configData->set($path, $deploymentConfig->get($path, $this->getDefaultValue($input))); ++ } ++ ++ return $configData; ++ } ++ ++ /** ++ * Returns default value by input key ++ * ++ * If default value is not set returns null ++ * ++ * @param string $inputKey ++ * @return mixed|null ++ */ ++ private function getDefaultValue(string $inputKey) ++ { ++ if (isset($this->defaultConfigValues[$inputKey])) { ++ return $this->defaultConfigValues[$inputKey]; ++ } else { ++ return null; ++ } ++ } ++} diff --git a/patches/MAGECLOUD-3054__add_zookeeper_and_flock_locks__2.3.0.patch b/patches/MAGECLOUD-3054__add_zookeeper_and_flock_locks__2.3.0.patch new file mode 100644 index 0000000..f364ba3 --- /dev/null +++ b/patches/MAGECLOUD-3054__add_zookeeper_and_flock_locks__2.3.0.patch @@ -0,0 +1,1055 @@ +diff -Naur a/app/etc/di.xml b/app/etc/di.xml +--- a/app/etc/di.xml ++++ b/app/etc/di.xml +@@ -38,7 +38,7 @@ + + + +- ++ + + + +diff -Naur a/vendor/magento/framework/Lock/Backend/FileLock.php b/vendor/magento/framework/Lock/Backend/FileLock.php +--- /dev/null ++++ b/vendor/magento/framework/Lock/Backend/FileLock.php +@@ -0,0 +1,194 @@ ++fileDriver = $fileDriver; ++ $this->path = rtrim($path, '/') . '/'; ++ ++ try { ++ if (!$this->fileDriver->isExists($this->path)) { ++ $this->fileDriver->createDirectory($this->path); ++ } ++ } catch (FileSystemException $exception) { ++ throw new RuntimeException( ++ new Phrase('Cannot create the directory for locks: %1', [$this->path]), ++ $exception ++ ); ++ } ++ } ++ ++ /** ++ * Acquires a lock by name ++ * ++ * @param string $name The lock name ++ * @param int $timeout Timeout in seconds. A negative timeout value means infinite timeout ++ * @return bool Returns true if the lock is acquired, otherwise returns false ++ * @throws RuntimeException Throws RuntimeException if cannot acquires the lock because FS problems ++ */ ++ public function lock(string $name, int $timeout = -1): bool ++ { ++ try { ++ $lockFile = $this->getLockPath($name); ++ $fileResource = $this->fileDriver->fileOpen($lockFile, 'w+'); ++ $skipDeadline = $timeout < 0; ++ $deadline = microtime(true) + $timeout; ++ ++ while (!$this->tryToLock($fileResource)) { ++ if (!$skipDeadline && $deadline <= microtime(true)) { ++ $this->fileDriver->fileClose($fileResource); ++ return false; ++ } ++ usleep($this->sleepCycle); ++ } ++ } catch (FileSystemException $exception) { ++ throw new RuntimeException(new Phrase('Cannot acquire a lock.'), $exception); ++ } ++ ++ $this->locks[$lockFile] = $fileResource; ++ return true; ++ } ++ ++ /** ++ * Checks if a lock exists by name ++ * ++ * @param string $name The lock name ++ * @return bool Returns true if the lock exists, otherwise returns false ++ * @throws RuntimeException Throws RuntimeException if cannot check that the lock exists ++ */ ++ public function isLocked(string $name): bool ++ { ++ $lockFile = $this->getLockPath($name); ++ $result = false; ++ ++ try { ++ if ($this->fileDriver->isExists($lockFile)) { ++ $fileResource = $this->fileDriver->fileOpen($lockFile, 'w+'); ++ if ($this->tryToLock($fileResource)) { ++ $result = false; ++ } else { ++ $result = true; ++ } ++ $this->fileDriver->fileClose($fileResource); ++ } ++ } catch (FileSystemException $exception) { ++ throw new RuntimeException(new Phrase('Cannot verify that the lock exists.'), $exception); ++ } ++ ++ return $result; ++ } ++ ++ /** ++ * Remove the lock by name ++ * ++ * @param string $name The lock name ++ * @return bool If the lock is removed returns true, otherwise returns false ++ */ ++ public function unlock(string $name): bool ++ { ++ $lockFile = $this->getLockPath($name); ++ ++ if (isset($this->locks[$lockFile]) && $this->tryToUnlock($this->locks[$lockFile])) { ++ unset($this->locks[$lockFile]); ++ return true; ++ } ++ ++ return false; ++ } ++ ++ /** ++ * Returns the full path to the lock file by name ++ * ++ * @param string $name The lock name ++ * @return string The path to the lock file ++ */ ++ private function getLockPath(string $name): string ++ { ++ return $this->path . $name; ++ } ++ ++ /** ++ * Tries to lock a file resource ++ * ++ * @param resource $resource The file resource ++ * @return bool If the lock is acquired returns true, otherwise returns false ++ */ ++ private function tryToLock($resource): bool ++ { ++ try { ++ return $this->fileDriver->fileLock($resource, LOCK_EX | LOCK_NB); ++ } catch (FileSystemException $exception) { ++ return false; ++ } ++ } ++ ++ /** ++ * Tries to unlock a file resource ++ * ++ * @param resource $resource The file resource ++ * @return bool If the lock is removed returns true, otherwise returns false ++ */ ++ private function tryToUnlock($resource): bool ++ { ++ try { ++ return $this->fileDriver->fileLock($resource, LOCK_UN | LOCK_NB); ++ } catch (FileSystemException $exception) { ++ return false; ++ } ++ } ++} +diff -Naur a/vendor/magento/framework/Lock/Backend/Zookeeper.php b/vendor/magento/framework/Lock/Backend/Zookeeper.php +--- /dev/null ++++ b/vendor/magento/framework/Lock/Backend/Zookeeper.php +@@ -0,0 +1,280 @@ ++\Zookeeper::PERM_ALL, 'scheme' => 'world', 'id' => 'anyone']]; ++ ++ /** ++ * The mapping list of the lock name with the full lock path ++ * ++ * @var array ++ */ ++ private $locks = []; ++ ++ /** ++ * The default path to storage locks ++ */ ++ const DEFAULT_PATH = '/magento/locks'; ++ ++ /** ++ * @param string $host The host to connect to Zookeeper ++ * @param string $path The base path to locks in Zookeeper ++ * @throws RuntimeException ++ */ ++ public function __construct(string $host, string $path = self::DEFAULT_PATH) ++ { ++ if (!$path) { ++ throw new RuntimeException( ++ new Phrase('The path needs to be a non-empty string.') ++ ); ++ } ++ ++ if (!$host) { ++ throw new RuntimeException( ++ new Phrase('The host needs to be a non-empty string.') ++ ); ++ } ++ ++ $this->host = $host; ++ $this->path = rtrim($path, '/') . '/'; ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * You can see the lock algorithm by the link ++ * @link https://zookeeper.apache.org/doc/r3.1.2/recipes.html#sc_recipes_Locks ++ * ++ * @throws RuntimeException ++ */ ++ public function lock(string $name, int $timeout = -1): bool ++ { ++ $skipDeadline = $timeout < 0; ++ $lockPath = $this->getFullPathToLock($name); ++ $deadline = microtime(true) + $timeout; ++ ++ if (!$this->checkAndCreateParentNode($lockPath)) { ++ throw new RuntimeException(new Phrase('Failed creating the path %1', [$lockPath])); ++ } ++ ++ $lockKey = $this->getProvider() ++ ->create($lockPath, '1', $this->acl, \Zookeeper::EPHEMERAL | \Zookeeper::SEQUENCE); ++ ++ if (!$lockKey) { ++ throw new RuntimeException(new Phrase('Failed creating lock %1', [$lockPath])); ++ } ++ ++ while ($this->isAnyLock($lockKey, $this->getIndex($lockKey))) { ++ if (!$skipDeadline && $deadline <= microtime(true)) { ++ $this->getProvider()->delete($lockKey); ++ return false; ++ } ++ ++ usleep($this->sleepCycle); ++ } ++ ++ $this->locks[$name] = $lockKey; ++ ++ return true; ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * @throws RuntimeException ++ */ ++ public function unlock(string $name): bool ++ { ++ if (!isset($this->locks[$name])) { ++ return false; ++ } ++ ++ return $this->getProvider()->delete($this->locks[$name]); ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * @throws RuntimeException ++ */ ++ public function isLocked(string $name): bool ++ { ++ return $this->isAnyLock($this->getFullPathToLock($name)); ++ } ++ ++ /** ++ * Gets full path to lock by its name ++ * ++ * @param string $name ++ * @return string ++ */ ++ private function getFullPathToLock(string $name): string ++ { ++ return $this->path . $name . '/' . $this->lockName; ++ } ++ ++ /** ++ * Initiolizes and returns Zookeeper provider ++ * ++ * @return \Zookeeper ++ * @throws RuntimeException ++ */ ++ private function getProvider(): \Zookeeper ++ { ++ if (!$this->zookeeper) { ++ $this->zookeeper = new \Zookeeper($this->host); ++ } ++ ++ $deadline = microtime(true) + $this->connectionTimeout; ++ while ($this->zookeeper->getState() != \Zookeeper::CONNECTED_STATE) { ++ if ($deadline <= microtime(true)) { ++ throw new RuntimeException(new Phrase('Zookeeper connection timed out!')); ++ } ++ usleep($this->sleepCycle); ++ } ++ ++ return $this->zookeeper; ++ } ++ ++ /** ++ * Checks and creates base path recursively ++ * ++ * @param string $path ++ * @return bool ++ * @throws RuntimeException ++ */ ++ private function checkAndCreateParentNode(string $path): bool ++ { ++ $path = dirname($path); ++ if ($this->getProvider()->exists($path)) { ++ return true; ++ } ++ ++ if (!$this->checkAndCreateParentNode($path)) { ++ return false; ++ } ++ ++ if ($this->getProvider()->create($path, '1', $this->acl)) { ++ return true; ++ } ++ ++ return $this->getProvider()->exists($path); ++ } ++ ++ /** ++ * Gets int increment of lock key ++ * ++ * @param string $key ++ * @return int|null ++ */ ++ private function getIndex(string $key) ++ { ++ if (!preg_match('/' . $this->lockName . '([0-9]+)$/', $key, $matches)) { ++ return null; ++ } ++ ++ return intval($matches[1]); ++ } ++ ++ /** ++ * Checks if there is any sequence node under parent of $fullKey. ++ * ++ * At first checks that the $fullKey node is present, if not - returns false. ++ * If $indexKey is non-null and there is a smaller index than $indexKey then returns true, ++ * otherwise returns false. ++ * ++ * @param string $fullKey The full path without any sequence info ++ * @param int|null $indexKey The index to compare ++ * @return bool ++ * @throws RuntimeException ++ */ ++ private function isAnyLock(string $fullKey, int $indexKey = null): bool ++ { ++ $parent = dirname($fullKey); ++ ++ if (!$this->getProvider()->exists($parent)) { ++ return false; ++ } ++ ++ $children = $this->getProvider()->getChildren($parent); ++ ++ if (null === $indexKey && !empty($children)) { ++ return true; ++ } ++ ++ foreach ($children as $childKey) { ++ $childIndex = $this->getIndex($childKey); ++ ++ if (null === $childIndex) { ++ continue; ++ } ++ ++ if ($childIndex < $indexKey) { ++ return true; ++ } ++ } ++ ++ return false; ++ } ++} +diff -Naur a/vendor/magento/framework/Lock/LockBackendFactory.php b/vendor/magento/framework/Lock/LockBackendFactory.php +--- /dev/null ++++ b/vendor/magento/framework/Lock/LockBackendFactory.php +@@ -0,0 +1,111 @@ ++ DatabaseLock::class, ++ self::LOCK_ZOOKEEPER => ZookeeperLock::class, ++ self::LOCK_CACHE => CacheLock::class, ++ self::LOCK_FILE => FileLock::class, ++ ]; ++ ++ /** ++ * @param ObjectManagerInterface $objectManager The Object Manager instance ++ * @param DeploymentConfig $deploymentConfig The Application deployment configuration ++ */ ++ public function __construct( ++ ObjectManagerInterface $objectManager, ++ DeploymentConfig $deploymentConfig ++ ) { ++ $this->objectManager = $objectManager; ++ $this->deploymentConfig = $deploymentConfig; ++ } ++ ++ /** ++ * Creates an instance of LockManagerInterface using information from deployment config ++ * ++ * @return LockManagerInterface ++ * @throws RuntimeException ++ */ ++ public function create(): LockManagerInterface ++ { ++ $provider = $this->deploymentConfig->get('lock/provider', self::LOCK_DB); ++ $config = $this->deploymentConfig->get('lock/config', []); ++ ++ if (!isset($this->lockers[$provider])) { ++ throw new RuntimeException(new Phrase('Unknown locks provider: %1', [$provider])); ++ } ++ ++ if (self::LOCK_ZOOKEEPER === $provider && !extension_loaded(self::LOCK_ZOOKEEPER)) { ++ throw new RuntimeException(new Phrase('php extension Zookeeper is not installed.')); ++ } ++ ++ return $this->objectManager->create($this->lockers[$provider], $config); ++ } ++} +diff -Naur a/vendor/magento/framework/Lock/Proxy.php b/vendor/magento/framework/Lock/Proxy.php +--- /dev/null ++++ b/vendor/magento/framework/Lock/Proxy.php +@@ -0,0 +1,83 @@ ++factory = $factory; ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * @throws RuntimeException ++ */ ++ public function isLocked(string $name): bool ++ { ++ return $this->getLocker()->isLocked($name); ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * @throws RuntimeException ++ */ ++ public function lock(string $name, int $timeout = -1): bool ++ { ++ return $this->getLocker()->lock($name, $timeout); ++ } ++ ++ /** ++ * @inheritdoc ++ * ++ * @throws RuntimeException ++ */ ++ public function unlock(string $name): bool ++ { ++ return $this->getLocker()->unlock($name); ++ } ++ ++ /** ++ * Gets LockManagerInterface implementation using Factory ++ * ++ * @return LockManagerInterface ++ * @throws RuntimeException ++ */ ++ private function getLocker(): LockManagerInterface ++ { ++ if (!$this->locker) { ++ $this->locker = $this->factory->create(); ++ } ++ ++ return $this->locker; ++ } ++} +diff -Naur a/setup/src/Magento/Setup/Model/ConfigOptionsList.php b/setup/src/Magento/Setup/Model/ConfigOptionsList.php +--- a/setup/src/Magento/Setup/Model/ConfigOptionsList.php ++++ b/setup/src/Magento/Setup/Model/ConfigOptionsList.php +@@ -50,7 +50,8 @@ class ConfigOptionsList implements ConfigOptionsListInterface + private $configOptionsListClasses = [ + \Magento\Setup\Model\ConfigOptionsList\Session::class, + \Magento\Setup\Model\ConfigOptionsList\Cache::class, +- \Magento\Setup\Model\ConfigOptionsList\PageCache::class ++ \Magento\Setup\Model\ConfigOptionsList\PageCache::class, ++ \Magento\Setup\Model\ConfigOptionsList\Lock::class, + ]; + + /** +diff -Naur a/setup/src/Magento/Setup/Model/ConfigOptionsList/Lock.php b/setup/src/Magento/Setup/Model/ConfigOptionsList/Lock.php +--- /dev/null ++++ b/setup/src/Magento/Setup/Model/ConfigOptionsList/Lock.php +@@ -0,0 +1,342 @@ ++ [ ++ self::INPUT_KEY_LOCK_PROVIDER => self::CONFIG_PATH_LOCK_PROVIDER, ++ self::INPUT_KEY_LOCK_DB_PREFIX => self::CONFIG_PATH_LOCK_DB_PREFIX, ++ ], ++ LockBackendFactory::LOCK_ZOOKEEPER => [ ++ self::INPUT_KEY_LOCK_PROVIDER => self::CONFIG_PATH_LOCK_PROVIDER, ++ self::INPUT_KEY_LOCK_ZOOKEEPER_HOST => self::CONFIG_PATH_LOCK_ZOOKEEPER_HOST, ++ self::INPUT_KEY_LOCK_ZOOKEEPER_PATH => self::CONFIG_PATH_LOCK_ZOOKEEPER_PATH, ++ ], ++ LockBackendFactory::LOCK_CACHE => [ ++ self::INPUT_KEY_LOCK_PROVIDER => self::CONFIG_PATH_LOCK_PROVIDER, ++ ], ++ LockBackendFactory::LOCK_FILE => [ ++ self::INPUT_KEY_LOCK_PROVIDER => self::CONFIG_PATH_LOCK_PROVIDER, ++ self::INPUT_KEY_LOCK_FILE_PATH => self::CONFIG_PATH_LOCK_FILE_PATH, ++ ], ++ ]; ++ ++ /** ++ * The list of default values ++ * ++ * @var array ++ */ ++ private $defaultConfigValues = [ ++ self::INPUT_KEY_LOCK_PROVIDER => LockBackendFactory::LOCK_DB, ++ self::INPUT_KEY_LOCK_DB_PREFIX => null, ++ self::INPUT_KEY_LOCK_ZOOKEEPER_PATH => ZookeeperLock::DEFAULT_PATH, ++ ]; ++ ++ /** ++ * @inheritdoc ++ */ ++ public function getOptions() ++ { ++ return [ ++ new SelectConfigOption( ++ self::INPUT_KEY_LOCK_PROVIDER, ++ SelectConfigOption::FRONTEND_WIZARD_SELECT, ++ $this->validLockProviders, ++ self::CONFIG_PATH_LOCK_PROVIDER, ++ 'Lock provider name', ++ LockBackendFactory::LOCK_DB ++ ), ++ new TextConfigOption( ++ self::INPUT_KEY_LOCK_DB_PREFIX, ++ TextConfigOption::FRONTEND_WIZARD_TEXT, ++ self::CONFIG_PATH_LOCK_DB_PREFIX, ++ 'Installation specific lock prefix to avoid lock conflicts' ++ ), ++ new TextConfigOption( ++ self::INPUT_KEY_LOCK_ZOOKEEPER_HOST, ++ TextConfigOption::FRONTEND_WIZARD_TEXT, ++ self::CONFIG_PATH_LOCK_ZOOKEEPER_HOST, ++ 'Host and port to connect to Zookeeper cluster. For example: 127.0.0.1:2181' ++ ), ++ new TextConfigOption( ++ self::INPUT_KEY_LOCK_ZOOKEEPER_PATH, ++ TextConfigOption::FRONTEND_WIZARD_TEXT, ++ self::CONFIG_PATH_LOCK_ZOOKEEPER_PATH, ++ 'The path where Zookeeper will save locks. The default path is: ' . ZookeeperLock::DEFAULT_PATH ++ ), ++ new TextConfigOption( ++ self::INPUT_KEY_LOCK_FILE_PATH, ++ TextConfigOption::FRONTEND_WIZARD_TEXT, ++ self::CONFIG_PATH_LOCK_FILE_PATH, ++ 'The path where file locks will be saved.' ++ ), ++ ]; ++ } ++ ++ /** ++ * @inheritdoc ++ */ ++ public function createConfig(array $options, DeploymentConfig $deploymentConfig) ++ { ++ $configData = new ConfigData(ConfigFilePool::APP_ENV); ++ $configData->setOverrideWhenSave(true); ++ $lockProvider = $this->getLockProvider($options, $deploymentConfig); ++ ++ $this->setDefaultConfiguration($configData, $deploymentConfig, $lockProvider); ++ ++ foreach ($this->mappingInputKeyToConfigPath[$lockProvider] as $input => $path) { ++ if (isset($options[$input])) { ++ $configData->set($path, $options[$input]); ++ } ++ } ++ ++ return $configData; ++ } ++ ++ /** ++ * @inheritdoc ++ */ ++ public function validate(array $options, DeploymentConfig $deploymentConfig) ++ { ++ $lockProvider = $this->getLockProvider($options, $deploymentConfig); ++ switch ($lockProvider) { ++ case LockBackendFactory::LOCK_ZOOKEEPER: ++ $errors = $this->validateZookeeperConfig($options, $deploymentConfig); ++ break; ++ case LockBackendFactory::LOCK_FILE: ++ $errors = $this->validateFileConfig($options, $deploymentConfig); ++ break; ++ case LockBackendFactory::LOCK_CACHE: ++ case LockBackendFactory::LOCK_DB: ++ $errors = []; ++ break; ++ default: ++ $errors[] = 'The lock provider ' . $lockProvider . ' does not exist.'; ++ } ++ ++ return $errors; ++ } ++ ++ /** ++ * Validates File locks configuration ++ * ++ * @param array $options ++ * @param DeploymentConfig $deploymentConfig ++ * @return array ++ */ ++ private function validateFileConfig(array $options, DeploymentConfig $deploymentConfig): array ++ { ++ $errors = []; ++ ++ $path = $options[self::INPUT_KEY_LOCK_FILE_PATH] ++ ?? $deploymentConfig->get( ++ self::CONFIG_PATH_LOCK_FILE_PATH, ++ $this->getDefaultValue(self::INPUT_KEY_LOCK_FILE_PATH) ++ ); ++ ++ if (!$path) { ++ $errors[] = 'The path needs to be a non-empty string.'; ++ } ++ ++ return $errors; ++ } ++ ++ /** ++ * Validates Zookeeper configuration ++ * ++ * @param array $options ++ * @param DeploymentConfig $deploymentConfig ++ * @return array ++ */ ++ private function validateZookeeperConfig(array $options, DeploymentConfig $deploymentConfig): array ++ { ++ $errors = []; ++ ++ if (!extension_loaded(LockBackendFactory::LOCK_ZOOKEEPER)) { ++ $errors[] = 'php extension Zookeeper is not installed.'; ++ } ++ ++ $host = $options[self::INPUT_KEY_LOCK_ZOOKEEPER_HOST] ++ ?? $deploymentConfig->get( ++ self::CONFIG_PATH_LOCK_ZOOKEEPER_HOST, ++ $this->getDefaultValue(self::INPUT_KEY_LOCK_ZOOKEEPER_HOST) ++ ); ++ $path = $options[self::INPUT_KEY_LOCK_ZOOKEEPER_PATH] ++ ?? $deploymentConfig->get( ++ self::CONFIG_PATH_LOCK_ZOOKEEPER_PATH, ++ $this->getDefaultValue(self::INPUT_KEY_LOCK_ZOOKEEPER_PATH) ++ ); ++ ++ if (!$path) { ++ $errors[] = 'Zookeeper path needs to be a non-empty string.'; ++ } ++ ++ if (!$host) { ++ $errors[] = 'Zookeeper host is should be set.'; ++ } ++ ++ return $errors; ++ } ++ ++ /** ++ * Returns the name of lock provider ++ * ++ * @param array $options ++ * @param DeploymentConfig $deploymentConfig ++ * @return string ++ */ ++ private function getLockProvider(array $options, DeploymentConfig $deploymentConfig): string ++ { ++ if (!isset($options[self::INPUT_KEY_LOCK_PROVIDER])) { ++ return (string) $deploymentConfig->get( ++ self::CONFIG_PATH_LOCK_PROVIDER, ++ $this->getDefaultValue(self::INPUT_KEY_LOCK_PROVIDER) ++ ); ++ } ++ ++ return (string) $options[self::INPUT_KEY_LOCK_PROVIDER]; ++ } ++ ++ /** ++ * Sets default configuration for locks ++ * ++ * @param ConfigData $configData ++ * @param DeploymentConfig $deploymentConfig ++ * @param string $lockProvider ++ * @return ConfigData ++ */ ++ private function setDefaultConfiguration( ++ ConfigData $configData, ++ DeploymentConfig $deploymentConfig, ++ string $lockProvider ++ ) { ++ foreach ($this->mappingInputKeyToConfigPath[$lockProvider] as $input => $path) { ++ $configData->set($path, $deploymentConfig->get($path, $this->getDefaultValue($input))); ++ } ++ ++ return $configData; ++ } ++ ++ /** ++ * Returns default value by input key ++ * ++ * If default value is not set returns null ++ * ++ * @param string $inputKey ++ * @return mixed|null ++ */ ++ private function getDefaultValue(string $inputKey) ++ { ++ if (isset($this->defaultConfigValues[$inputKey])) { ++ return $this->defaultConfigValues[$inputKey]; ++ } else { ++ return null; ++ } ++ } ++} diff --git a/patches/MAGECLOUD-3611__multi_thread_scd__2.2.0.patch b/patches/MAGECLOUD-3611__multi_thread_scd__2.2.0.patch new file mode 100644 index 0000000..171e72d --- /dev/null +++ b/patches/MAGECLOUD-3611__multi_thread_scd__2.2.0.patch @@ -0,0 +1,63 @@ +diff -Naur a/vendor/magento/module-deploy/Process/Queue.php b/vendor/magento/module-deploy/Process/Queue.php +index e5e10c8f54a..85ef6514432 100644 +--- a/vendor/magento/module-deploy/Process/Queue.php ++++ b/vendor/magento/module-deploy/Process/Queue.php +@@ -291,12 +291,30 @@ class Queue + { + if ($this->isCanBeParalleled()) { + if ($package->getState() === null) { +- $pid = pcntl_waitpid($this->getPid($package), $status, WNOHANG); +- if ($pid === $this->getPid($package)) { ++ $pid = $this->getPid($package); ++ if ($pid === null) { ++ return false; ++ } ++ $result = pcntl_waitpid($pid, $status, WNOHANG); ++ if ($result === $pid) { + $package->setState(Package::STATE_COMPLETED); ++ $exitStatus = pcntl_wexitstatus($status); ++ $this->logger->info( ++ "Exited: " . $package->getPath() . "(status: $exitStatus)", ++ [ ++ 'process' => $package->getPath(), ++ 'status' => $exitStatus, ++ ] ++ ); + + unset($this->inProgress[$package->getPath()]); + return pcntl_wexitstatus($status) === 0; ++ } elseif ($result === -1) { ++ $errno = pcntl_errno(); ++ $strerror = pcntl_strerror($errno); ++ throw new \RuntimeException( ++ "Error encountered checking child process status (PID: $pid): $strerror (errno: $errno)" ++ ); + } + return false; + } +@@ -333,11 +351,23 @@ class Queue + public function __destruct() + { + foreach ($this->inProgress as $package) { +- if (pcntl_waitpid($this->getPid($package), $status) === -1) { ++ $pid = $this->getPid($package); ++ $this->logger->info( ++ "Reaping child process: {$package->getPath()} (PID: $pid)", ++ [ ++ 'process' => $package->getPath(), ++ 'pid' => $pid, ++ ] ++ ); ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ if (pcntl_waitpid($pid, $status) === -1) { ++ $errno = pcntl_errno(); ++ $strerror = pcntl_strerror($errno); + throw new \RuntimeException( +- 'Error while waiting for package deployed: ' . $this->getPid($package) . '; Status: ' . $status ++ "Error encountered waiting for child process (PID: $pid): $strerror (errno: $errno)" + ); + } ++ + } + } + } diff --git a/patches/MAGECLOUD-3611__multi_thread_scd__2.2.4.patch b/patches/MAGECLOUD-3611__multi_thread_scd__2.2.4.patch new file mode 100644 index 0000000..5425daa --- /dev/null +++ b/patches/MAGECLOUD-3611__multi_thread_scd__2.2.4.patch @@ -0,0 +1,245 @@ +diff -Naur a/vendor/magento/module-deploy/Process/Queue.php b/vendor/magento/module-deploy/Process/Queue.php +index d8089457ce5b..ca75bf1acb73 100644 +--- a/vendor/magento/module-deploy/Process/Queue.php ++++ b/vendor/magento/module-deploy/Process/Queue.php +@@ -3,14 +3,16 @@ + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ ++declare(strict_types=1); ++ + namespace Magento\Deploy\Process; + + use Magento\Deploy\Package\Package; + use Magento\Deploy\Service\DeployPackage; + use Magento\Framework\App\ResourceConnection; +-use Psr\Log\LoggerInterface; + use Magento\Framework\App\State as AppState; + use Magento\Framework\Locale\ResolverInterface as LocaleResolver; ++use Psr\Log\LoggerInterface; + + /** + * Deployment Queue +@@ -125,6 +127,8 @@ public function __construct( + } + + /** ++ * Adds deployment package. ++ * + * @param Package $package + * @param Package[] $dependencies + * @return bool true on success +@@ -140,6 +144,8 @@ public function add(Package $package, array $dependencies = []) + } + + /** ++ * Returns packages array. ++ * + * @return Package[] + */ + public function getPackages() +@@ -159,9 +165,11 @@ public function process() + $packages = $this->packages; + while (count($packages) && $this->checkTimeout()) { + foreach ($packages as $name => $packageJob) { ++ // Unsets each member of $packages array (passed by reference) as each is executed + $this->assertAndExecute($name, $packages, $packageJob); + } + $this->logger->info('.'); ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction + sleep(3); + foreach ($this->inProgress as $name => $package) { + if ($this->isDeployed($package)) { +@@ -182,8 +190,6 @@ public function process() + * @param array $packages + * @param array $packageJob + * @return void +- * +- * @SuppressWarnings(PHPMD.CyclomaticComplexity) + */ + private function assertAndExecute($name, array & $packages, array $packageJob) + { +@@ -207,13 +213,23 @@ private function assertAndExecute($name, array & $packages, array $packageJob) + } + } + } ++ $this->executePackage($package, $name, $packages, $dependenciesNotFinished); ++ } + ++ /** ++ * Executes deployment package. ++ * ++ * @param Package $package ++ * @param string $name ++ * @param array $packages ++ * @param bool $dependenciesNotFinished ++ * @return void ++ */ ++ private function executePackage(Package $package, string $name, array &$packages, bool $dependenciesNotFinished) ++ { + if (!$dependenciesNotFinished + && !$this->isDeployed($package) +- && ( +- $this->maxProcesses < 2 +- || (count($this->inProgress) < $this->maxProcesses) +- ) ++ && ($this->maxProcesses < 2 || (count($this->inProgress) < $this->maxProcesses)) + ) { + unset($packages[$name]); + $this->execute($package); +@@ -234,6 +250,7 @@ private function awaitForAllProcesses() + } + } + $this->logger->info('.'); ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction + sleep(5); + } + if ($this->isCanBeParalleled()) { +@@ -243,6 +260,8 @@ private function awaitForAllProcesses() + } + + /** ++ * Checks if can be parallel. ++ * + * @return bool + */ + private function isCanBeParalleled() +@@ -251,9 +270,12 @@ private function isCanBeParalleled() + } + + /** ++ * Executes the process. ++ * + * @param Package $package + * @return bool true on success for main process and exit for child process + * @SuppressWarnings(PHPMD.ExitExpression) ++ * @throws \RuntimeException + */ + private function execute(Package $package) + { +@@ -281,6 +303,7 @@ function () use ($package) { + ); + + if ($this->isCanBeParalleled()) { ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction + $pid = pcntl_fork(); + if ($pid === -1) { + throw new \RuntimeException('Unable to fork a new process'); +@@ -295,6 +318,7 @@ function () use ($package) { + // process child process + $this->inProgress = []; + $this->deployPackageService->deploy($package, $this->options, true); ++ // phpcs:ignore Magento2.Security.LanguageConstruct.ExitUsage + exit(0); + } else { + $this->deployPackageService->deploy($package, $this->options); +@@ -303,6 +327,8 @@ function () use ($package) { + } + + /** ++ * Checks if package is deployed. ++ * + * @param Package $package + * @return bool + */ +@@ -310,12 +336,41 @@ private function isDeployed(Package $package) + { + if ($this->isCanBeParalleled()) { + if ($package->getState() === null) { +- $pid = pcntl_waitpid($this->getPid($package), $status, WNOHANG); +- if ($pid === $this->getPid($package)) { ++ $pid = $this->getPid($package); ++ ++ // When $pid comes back as null the child process for this package has not yet started; prevents both ++ // hanging until timeout expires (which was behaviour in 2.2.x) and the type error from strict_types ++ if ($pid === null) { ++ return false; ++ } ++ ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ $result = pcntl_waitpid($pid, $status, WNOHANG); ++ if ($result === $pid) { + $package->setState(Package::STATE_COMPLETED); ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ $exitStatus = pcntl_wexitstatus($status); ++ ++ $this->logger->info( ++ "Exited: " . $package->getPath() . "(status: $exitStatus)", ++ [ ++ 'process' => $package->getPath(), ++ 'status' => $exitStatus, ++ ] ++ ); + + unset($this->inProgress[$package->getPath()]); ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction + return pcntl_wexitstatus($status) === 0; ++ } elseif ($result === -1) { ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ $errno = pcntl_errno(); ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ $strerror = pcntl_strerror($errno); ++ ++ throw new \RuntimeException( ++ "Error encountered checking child process status (PID: $pid): $strerror (errno: $errno)" ++ ); + } + return false; + } +@@ -324,17 +379,19 @@ private function isDeployed(Package $package) + } + + /** ++ * Returns process ID or null if not found. ++ * + * @param Package $package + * @return int|null + */ + private function getPid(Package $package) + { +- return isset($this->processIds[$package->getPath()]) +- ? $this->processIds[$package->getPath()] +- : null; ++ return $this->processIds[$package->getPath()] ?? null; + } + + /** ++ * Checks timeout. ++ * + * @return bool + */ + private function checkTimeout() +@@ -347,14 +404,31 @@ private function checkTimeout() + * + * Protect against zombie process + * ++ * @throws \RuntimeException ++ * @SuppressWarnings(PHPMD.UnusedLocalVariable) + * @return void + */ + public function __destruct() + { + foreach ($this->inProgress as $package) { +- if (pcntl_waitpid($this->getPid($package), $status) === -1) { ++ $pid = $this->getPid($package); ++ $this->logger->info( ++ "Reaping child process: {$package->getPath()} (PID: $pid)", ++ [ ++ 'process' => $package->getPath(), ++ 'pid' => $pid, ++ ] ++ ); ++ ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ if (pcntl_waitpid($pid, $status) === -1) { ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ $errno = pcntl_errno(); ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ $strerror = pcntl_strerror($errno); ++ + throw new \RuntimeException( +- 'Error while waiting for package deployed: ' . $this->getPid($package) . '; Status: ' . $status ++ "Error encountered waiting for child process (PID: $pid): $strerror (errno: $errno)" + ); + } + } diff --git a/patches/MAGECLOUD-3611__multi_thread_scd__2.3.0.patch b/patches/MAGECLOUD-3611__multi_thread_scd__2.3.0.patch new file mode 100644 index 0000000..3ba98d4 --- /dev/null +++ b/patches/MAGECLOUD-3611__multi_thread_scd__2.3.0.patch @@ -0,0 +1,63 @@ +diff -Naur a/vendor/magento/module-deploy/Process/Queue.php b/vendor/magento/module-deploy/Process/Queue.php +index e5e10c8f54a..85ef6514432 100644 +--- a/vendor/magento/module-deploy/Process/Queue.php ++++ b/vendor/magento/module-deploy/Process/Queue.php +@@ -291,12 +291,30 @@ class Queue + { + if ($this->isCanBeParalleled()) { + if ($package->getState() === null) { +- $pid = pcntl_waitpid($this->getPid($package), $status, WNOHANG); +- if ($pid === $this->getPid($package)) { ++ $pid = $this->getPid($package); ++ if ($pid === null) { ++ return false; ++ } ++ $result = pcntl_waitpid($pid, $status, WNOHANG); ++ if ($result === $pid) { + $package->setState(Package::STATE_COMPLETED); ++ $exitStatus = pcntl_wexitstatus($status); ++ $this->logger->info( ++ "Exited: " . $package->getPath() . "(status: $exitStatus)", ++ [ ++ 'process' => $package->getPath(), ++ 'status' => $exitStatus, ++ ] ++ ); + + unset($this->inProgress[$package->getPath()]); + return pcntl_wexitstatus($status) === 0; ++ } elseif ($result === -1) { ++ $errno = pcntl_errno(); ++ $strerror = pcntl_strerror($errno); ++ throw new \RuntimeException( ++ "Error encountered checking child process status (PID: $pid): $strerror (errno: $errno)" ++ ); + } + return false; + } +@@ -333,11 +351,23 @@ class Queue + public function __destruct() + { + foreach ($this->inProgress as $package) { +- if (pcntl_waitpid($this->getPid($package), $status) === -1) { ++ $pid = $this->getPid($package); ++ $this->logger->info( ++ "Reaping child process: {$package->getPath()} (PID: $pid)", ++ [ ++ 'process' => $package->getPath(), ++ 'pid' => $pid, ++ ] ++ ); ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ if (pcntl_waitpid($pid, $status) === -1) { ++ $errno = pcntl_errno(); ++ $strerror = pcntl_strerror($errno); + throw new \RuntimeException( +- 'Error while waiting for package deployed: ' . $this->getPid($package) . '; Status: ' . $status ++ "Error encountered waiting for child process (PID: $pid): $strerror (errno: $errno)" + ); + } ++ + } + } + } diff --git a/patches/MAGECLOUD-3611__multi_thread_scd__2.3.2.patch b/patches/MAGECLOUD-3611__multi_thread_scd__2.3.2.patch new file mode 100644 index 0000000..4bb7aa1 --- /dev/null +++ b/patches/MAGECLOUD-3611__multi_thread_scd__2.3.2.patch @@ -0,0 +1,71 @@ +diff -Naur a/vendor/magento/module-deploy/Process/Queue.php b/vendor/magento/module-deploy/Process/Queue.php +index d7bb816e61c..c7fe02a4b02 100644 +--- a/vendor/magento/module-deploy/Process/Queue.php ++++ b/vendor/magento/module-deploy/Process/Queue.php +@@ -338,14 +338,37 @@ class Queue + { + if ($this->isCanBeParalleled()) { + if ($package->getState() === null) { ++ $pid = $this->getPid($package); ++ // When $pid comes back as null the child process for this package has not yet started; prevents both ++ // hanging until timeout expires (which was behaviour in 2.2.x) and the type error from strict_types ++ if ($pid === null) { ++ return false; ++ } + // phpcs:ignore Magento2.Functions.DiscouragedFunction +- $pid = pcntl_waitpid($this->getPid($package) ?? 0, $status, WNOHANG); +- if ($pid === $this->getPid($package)) { ++ $result = pcntl_waitpid($pid, $status, WNOHANG); ++ if ($result === $pid) { + $package->setState(Package::STATE_COMPLETED); ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ $exitStatus = pcntl_wexitstatus($status); ++ $this->logger->info( ++ "Exited: " . $package->getPath() . "(status: $exitStatus)", ++ [ ++ 'process' => $package->getPath(), ++ 'status' => $exitStatus, ++ ] ++ ); + + unset($this->inProgress[$package->getPath()]); + // phpcs:ignore Magento2.Functions.DiscouragedFunction + return pcntl_wexitstatus($status) === 0; ++ } elseif ($result === -1) { ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ $errno = pcntl_errno(); ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ $strerror = pcntl_strerror($errno); ++ throw new \RuntimeException( ++ "Error encountered checking child process status (PID: $pid): $strerror (errno: $errno)" ++ ); + } + return false; + } +@@ -385,10 +408,24 @@ class Queue + public function __destruct() + { + foreach ($this->inProgress as $package) { ++ $pid = $this->getPid($package); ++ $this->logger->info( ++ "Reaping child process: {$package->getPath()} (PID: $pid)", ++ [ ++ 'process' => $package->getPath(), ++ 'pid' => $pid, ++ ] ++ ); ++ + // phpcs:ignore Magento2.Functions.DiscouragedFunction +- if (pcntl_waitpid($this->getPid($package), $status) === -1) { ++ if (pcntl_waitpid($pid, $status) === -1) { ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ $errno = pcntl_errno(); ++ // phpcs:ignore Magento2.Functions.DiscouragedFunction ++ $strerror = pcntl_strerror($errno); ++ + throw new \RuntimeException( +- 'Error while waiting for package deployed: ' . $this->getPid($package) . '; Status: ' . $status ++ "Error encountered waiting for child process (PID: $pid): $strerror (errno: $errno)" + ); + } + } diff --git a/patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.2.0.patch b/patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.2.0.patch new file mode 100644 index 0000000..e8af178 --- /dev/null +++ b/patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.2.0.patch @@ -0,0 +1,32 @@ +diff -Nuar a/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php b/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php +--- a/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php ++++ b/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php +@@ -77,6 +77,7 @@ class UpgradeCommand extends AbstractSetupCommand + protected function execute(InputInterface $input, OutputInterface $output) + { + try { ++ $resultCode = \Magento\Framework\Console\Cli::RETURN_SUCCESS; + $keepGenerated = $input->getOption(self::INPUT_KEY_KEEP_GENERATED); + $installer = $this->installerFactory->create(new ConsoleLogger($output)); + $installer->updateModulesSequence($keepGenerated); +@@ -87,10 +88,10 @@ class UpgradeCommand extends AbstractSetupCommand + $importConfigCommand = $this->getApplication()->find(ConfigImportCommand::COMMAND_NAME); + $arrayInput = new ArrayInput([]); + $arrayInput->setInteractive($input->isInteractive()); +- $importConfigCommand->run($arrayInput, $output); ++ $resultCode = $importConfigCommand->run($arrayInput, $output); + } + +- if (!$keepGenerated) { ++ if ($resultCode !== \Magento\Framework\Console\Cli::RETURN_FAILURE && !$keepGenerated) { + $output->writeln( + 'Please re-run Magento compile command. Use the command "setup:di:compile"' + ); +@@ -100,6 +101,6 @@ class UpgradeCommand extends AbstractSetupCommand + return \Magento\Framework\Console\Cli::RETURN_FAILURE; + } + +- return \Magento\Framework\Console\Cli::RETURN_SUCCESS; ++ return $resultCode; + } + } diff --git a/patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.2.1.patch b/patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.2.1.patch new file mode 100644 index 0000000..78d77d8 --- /dev/null +++ b/patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.2.1.patch @@ -0,0 +1,33 @@ +diff -Nuar a/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php b/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php +--- a/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php ++++ b/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php +@@ -87,6 +87,7 @@ protected function configure() + protected function execute(InputInterface $input, OutputInterface $output) + { + try { ++ $resultCode = \Magento\Framework\Console\Cli::RETURN_SUCCESS; + $keepGenerated = $input->getOption(self::INPUT_KEY_KEEP_GENERATED); + $installer = $this->installerFactory->create(new ConsoleLogger($output)); + $installer->updateModulesSequence($keepGenerated); +@@ -97,10 +98,11 @@ protected function execute(InputInterface $input, OutputInterface $output) + $importConfigCommand = $this->getApplication()->find(ConfigImportCommand::COMMAND_NAME); + $arrayInput = new ArrayInput([]); + $arrayInput->setInteractive($input->isInteractive()); +- $importConfigCommand->run($arrayInput, $output); ++ $resultCode = $importConfigCommand->run($arrayInput, $output); + } + +- if (!$keepGenerated && $this->appState->getMode() === AppState::MODE_PRODUCTION) { ++ if ($resultCode !== \Magento\Framework\Console\Cli::RETURN_FAILURE ++ && !$keepGenerated && $this->appState->getMode() === AppState::MODE_PRODUCTION) { + $output->writeln( + 'Please re-run Magento compile command. Use the command "setup:di:compile"' + ); +@@ -110,6 +112,6 @@ protected function execute(InputInterface $input, OutputInterface $output) + return \Magento\Framework\Console\Cli::RETURN_FAILURE; + } + +- return \Magento\Framework\Console\Cli::RETURN_SUCCESS; ++ return $resultCode; + } + } diff --git a/patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.3.0.patch b/patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.3.0.patch new file mode 100644 index 0000000..e3b06dd --- /dev/null +++ b/patches/MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.3.0.patch @@ -0,0 +1,15 @@ +diff -Nuar a/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php b/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php +--- a/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php ++++ b/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php +@@ -126,7 +126,10 @@ protected function execute(InputInterface $input, OutputInterface $output) + $importConfigCommand = $this->getApplication()->find(ConfigImportCommand::COMMAND_NAME); + $arrayInput = new ArrayInput([]); + $arrayInput->setInteractive($input->isInteractive()); +- $importConfigCommand->run($arrayInput, $output); ++ $result = $importConfigCommand->run($arrayInput, $output); ++ if ($result === \Magento\Framework\Console\Cli::RETURN_FAILURE) { ++ throw new \Magento\Framework\Exception\RuntimeException(__('%1 failed. See previous output.', ConfigImportCommand::COMMAND_NAME)); ++ } + } + + if (!$keepGenerated && $this->appState->getMode() === AppState::MODE_PRODUCTION) { diff --git a/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.5.patch b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.5.patch new file mode 100644 index 0000000..a10280b --- /dev/null +++ b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.5.patch @@ -0,0 +1,573 @@ +diff -Nuar a/vendor/magento/framework/Lock/Backend/Database.php b/vendor/magento/framework/Lock/Backend/Database.php +--- a/vendor/magento/framework/Lock/Backend/Database.php ++++ b/vendor/magento/framework/Lock/Backend/Database.php +@@ -3,8 +3,8 @@ + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +- + declare(strict_types=1); ++ + namespace Magento\Framework\Lock\Backend; + + use Magento\Framework\App\DeploymentConfig; +@@ -14,20 +14,44 @@ use Magento\Framework\Exception\AlreadyExistsException; + use Magento\Framework\Exception\InputException; + use Magento\Framework\Phrase; + ++/** ++ * Implementation of the lock manager on the basis of MySQL. ++ */ + class Database implements \Magento\Framework\Lock\LockManagerInterface + { +- /** @var ResourceConnection */ ++ /** ++ * Max time for lock is 1 week ++ * ++ * MariaDB does not support negative timeout value to get infinite timeout, ++ * so we set 1 week for lock timeout ++ */ ++ const MAX_LOCK_TIME = 604800; ++ ++ /** ++ * @var ResourceConnection ++ */ + private $resource; + +- /** @var DeploymentConfig */ ++ /** ++ * @var DeploymentConfig ++ */ + private $deploymentConfig; + +- /** @var string Lock prefix */ ++ /** ++ * @var string Lock prefix ++ */ + private $prefix; + +- /** @var string|false Holds current lock name if set, otherwise false */ ++ /** ++ * @var string|false Holds current lock name if set, otherwise false ++ */ + private $currentLock = false; + ++ /** ++ * @param ResourceConnection $resource ++ * @param DeploymentConfig $deploymentConfig ++ * @param string|null $prefix ++ */ + public function __construct( + ResourceConnection $resource, + DeploymentConfig $deploymentConfig, +@@ -46,9 +70,13 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @return bool + * @throws InputException + * @throws AlreadyExistsException ++ * @throws \Zend_Db_Statement_Exception + */ + public function lock(string $name, int $timeout = -1): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return true; ++ }; + $name = $this->addPrefix($name); + + /** +@@ -59,7 +87,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + if ($this->currentLock) { + throw new AlreadyExistsException( + new Phrase( +- 'Current connection is already holding lock for $1, only single lock allowed', ++ 'Current connection is already holding lock for %1, only single lock allowed', + [$this->currentLock] + ) + ); +@@ -67,7 +95,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + + $result = (bool)$this->resource->getConnection()->query( + "SELECT GET_LOCK(?, ?);", +- [(string)$name, (int)$timeout] ++ [$name, $timeout < 0 ? self::MAX_LOCK_TIME : $timeout] + )->fetchColumn(); + + if ($result === true) { +@@ -83,9 +111,14 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @param string $name lock name + * @return bool + * @throws InputException ++ * @throws \Zend_Db_Statement_Exception + */ + public function unlock(string $name): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return true; ++ }; ++ + $name = $this->addPrefix($name); + + $result = (bool)$this->resource->getConnection()->query( +@@ -106,14 +139,19 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @param string $name lock name + * @return bool + * @throws InputException ++ * @throws \Zend_Db_Statement_Exception + */ + public function isLocked(string $name): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return false; ++ }; ++ + $name = $this->addPrefix($name); + + return (bool)$this->resource->getConnection()->query( + "SELECT IS_USED_LOCK(?);", +- [(string)$name] ++ [$name] + )->fetchColumn(); + } + +@@ -123,7 +161,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * Limited to 64 characters in MySQL. + * + * @param string $name +- * @return string $name ++ * @return string + * @throws InputException + */ + private function addPrefix(string $name): string +diff -Nuar a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +--- a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php ++++ b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +@@ -11,7 +11,7 @@ use Symfony\Component\Console\Input\InputInterface; + use Symfony\Component\Console\Input\InputOption; + use Symfony\Component\Console\Output\OutputInterface; + use Magento\Framework\MessageQueue\ConsumerFactory; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Command for starting MessageQueue consumers. +@@ -22,6 +22,7 @@ class StartConsumerCommand extends Command + const OPTION_NUMBER_OF_MESSAGES = 'max-messages'; + const OPTION_BATCH_SIZE = 'batch-size'; + const OPTION_AREACODE = 'area-code'; ++ const OPTION_SINGLE_THREAD = 'single-thread'; + const PID_FILE_PATH = 'pid-file-path'; + const COMMAND_QUEUE_CONSUMERS_START = 'queue:consumers:start'; + +@@ -36,9 +37,9 @@ class StartConsumerCommand extends Command + private $appState; + + /** +- * @var PidConsumerManager ++ * @var LockManagerInterface + */ +- private $pidConsumerManager; ++ private $lockManager; + + /** + * StartConsumerCommand constructor. +@@ -47,23 +48,23 @@ class StartConsumerCommand extends Command + * @param \Magento\Framework\App\State $appState + * @param ConsumerFactory $consumerFactory + * @param string $name +- * @param PidConsumerManager $pidConsumerManager ++ * @param LockManagerInterface $lockManager + */ + public function __construct( + \Magento\Framework\App\State $appState, + ConsumerFactory $consumerFactory, + $name = null, +- PidConsumerManager $pidConsumerManager = null ++ LockManagerInterface $lockManager = null + ) { + $this->appState = $appState; + $this->consumerFactory = $consumerFactory; +- $this->pidConsumerManager = $pidConsumerManager ?: \Magento\Framework\App\ObjectManager::getInstance() +- ->get(PidConsumerManager::class); ++ $this->lockManager = $lockManager ?: \Magento\Framework\App\ObjectManager::getInstance() ++ ->get(LockManagerInterface::class); + parent::__construct($name); + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function execute(InputInterface $input, OutputInterface $output) + { +@@ -71,30 +72,36 @@ class StartConsumerCommand extends Command + $numberOfMessages = $input->getOption(self::OPTION_NUMBER_OF_MESSAGES); + $batchSize = (int)$input->getOption(self::OPTION_BATCH_SIZE); + $areaCode = $input->getOption(self::OPTION_AREACODE); +- $pidFilePath = $input->getOption(self::PID_FILE_PATH); + +- if ($pidFilePath && $this->pidConsumerManager->isRun($pidFilePath)) { +- $output->writeln('Consumer with the same PID is running'); +- return \Magento\Framework\Console\Cli::RETURN_FAILURE; ++ if ($input->getOption(self::PID_FILE_PATH)) { ++ $input->setOption(self::OPTION_SINGLE_THREAD, true); + } + +- if ($pidFilePath) { +- $this->pidConsumerManager->savePid($pidFilePath); ++ $singleThread = $input->getOption(self::OPTION_SINGLE_THREAD); ++ ++ if ($singleThread && $this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore ++ $output->writeln('Consumer with the same name is running'); ++ return \Magento\Framework\Console\Cli::RETURN_FAILURE; + } + +- if ($areaCode !== null) { +- $this->appState->setAreaCode($areaCode); +- } else { +- $this->appState->setAreaCode('global'); ++ if ($singleThread) { ++ $this->lockManager->lock(md5($consumerName)); //phpcs:ignore + } + ++ $this->appState->setAreaCode($areaCode ?? 'global'); ++ + $consumer = $this->consumerFactory->get($consumerName, $batchSize); + $consumer->process($numberOfMessages); ++ ++ if ($singleThread) { ++ $this->lockManager->unlock(md5($consumerName)); //phpcs:ignore ++ } ++ + return \Magento\Framework\Console\Cli::RETURN_SUCCESS; + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function configure() + { +@@ -125,11 +132,17 @@ class StartConsumerCommand extends Command + 'The preferred area (global, adminhtml, etc...) ' + . 'default is global.' + ); ++ $this->addOption( ++ self::OPTION_SINGLE_THREAD, ++ null, ++ InputOption::VALUE_NONE, ++ 'This option prevents running multiple copies of one consumer simultaneously.' ++ ); + $this->addOption( + self::PID_FILE_PATH, + null, + InputOption::VALUE_REQUIRED, +- 'The file path for saving PID' ++ 'The file path for saving PID (This option is deprecated, use --single-thread instead)' + ); + $this->setHelp( + <<%command.full_name% someConsumer --area-code='adminhtml' ++ ++To do not run multiple copies of one consumer simultaneously: ++ ++ %command.full_name% someConsumer --single-thread' + +-To save PID enter path: ++To save PID enter path (This option is deprecated, use --single-thread instead): + + %command.full_name% someConsumer --pid-file-path='/var/someConsumer.pid' + HELP +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php ++++ b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +@@ -5,22 +5,21 @@ + */ + namespace Magento\MessageQueue\Model\Cron; + ++use Magento\Framework\App\ObjectManager; ++use Magento\Framework\MessageQueue\ConnectionTypeResolver; ++use Magento\Framework\MessageQueue\Consumer\Config\ConsumerConfigItemInterface; + use Magento\Framework\ShellInterface; + use Magento\Framework\MessageQueue\Consumer\ConfigInterface as ConsumerConfigInterface; + use Magento\Framework\App\DeploymentConfig; ++use Psr\Log\LoggerInterface; + use Symfony\Component\Process\PhpExecutableFinder; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Class for running consumers processes by cron + */ + class ConsumersRunner + { +- /** +- * Extension of PID file +- */ +- const PID_FILE_EXT = '.pid'; +- + /** + * Shell command line wrapper for executing command in background + * +@@ -50,11 +49,21 @@ class ConsumersRunner + private $phpExecutableFinder; + + /** +- * The class for checking status of process by PID ++ * @var ConnectionTypeResolver ++ */ ++ private $mqConnectionTypeResolver; ++ ++ /** ++ * @var LoggerInterface ++ */ ++ private $logger; ++ ++ /** ++ * Lock Manager + * +- * @var PidConsumerManager ++ * @var LockManagerInterface + */ +- private $pidConsumerManager; ++ private $lockManager; + + /** + * @param PhpExecutableFinder $phpExecutableFinder The executable finder specifically designed +@@ -62,20 +71,28 @@ class ConsumersRunner + * @param ConsumerConfigInterface $consumerConfig The consumer config provider + * @param DeploymentConfig $deploymentConfig The application deployment configuration + * @param ShellInterface $shellBackground The shell command line wrapper for executing command in background +- * @param PidConsumerManager $pidConsumerManager The class for checking status of process by PID ++ * @param LockManagerInterface $lockManager The lock manager ++ * @param ConnectionTypeResolver $mqConnectionTypeResolver Consumer connection resolver ++ * @param LoggerInterface $logger Logger + */ + public function __construct( + PhpExecutableFinder $phpExecutableFinder, + ConsumerConfigInterface $consumerConfig, + DeploymentConfig $deploymentConfig, + ShellInterface $shellBackground, +- PidConsumerManager $pidConsumerManager ++ LockManagerInterface $lockManager, ++ ConnectionTypeResolver $mqConnectionTypeResolver = null, ++ LoggerInterface $logger = null + ) { + $this->phpExecutableFinder = $phpExecutableFinder; + $this->consumerConfig = $consumerConfig; + $this->deploymentConfig = $deploymentConfig; + $this->shellBackground = $shellBackground; +- $this->pidConsumerManager = $pidConsumerManager; ++ $this->lockManager = $lockManager; ++ $this->mqConnectionTypeResolver = $mqConnectionTypeResolver ++ ?: ObjectManager::getInstance()->get(ConnectionTypeResolver::class); ++ $this->logger = $logger ++ ?: ObjectManager::getInstance()->get(LoggerInterface::class); + } + + /** +@@ -94,15 +111,13 @@ class ConsumersRunner + $php = $this->phpExecutableFinder->find() ?: 'php'; + + foreach ($this->consumerConfig->getConsumers() as $consumer) { +- $consumerName = $consumer->getName(); +- +- if (!$this->canBeRun($consumerName, $allowedConsumers)) { ++ if (!$this->canBeRun($consumer, $allowedConsumers)) { + continue; + } + + $arguments = [ +- $consumerName, +- '--pid-file-path=' . $this->getPidFilePath($consumerName), ++ $consumer->getName(), ++ '--single-thread' + ]; + + if ($maxMessages) { +@@ -119,26 +134,38 @@ class ConsumersRunner + /** + * Checks that the consumer can be run + * +- * @param string $consumerName The consumer name ++ * @param ConsumerConfigItemInterface $consumerConfig The consumer config + * @param array $allowedConsumers The list of allowed consumers + * If $allowedConsumers is empty it means that all consumers are allowed + * @return bool Returns true if the consumer can be run ++ * @throws \Magento\Framework\Exception\FileSystemException + */ +- private function canBeRun($consumerName, array $allowedConsumers = []) ++ private function canBeRun(ConsumerConfigItemInterface $consumerConfig, array $allowedConsumers = []): bool + { +- $allowed = empty($allowedConsumers) ?: in_array($consumerName, $allowedConsumers); ++ $consumerName = $consumerConfig->getName(); ++ if (!empty($allowedConsumers) && !in_array($consumerName, $allowedConsumers)) { ++ return false; ++ } + +- return $allowed && !$this->pidConsumerManager->isRun($this->getPidFilePath($consumerName)); +- } ++ if ($this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore ++ return false; ++ } + +- /** +- * Returns default path to file with PID by consumers name +- * +- * @param string $consumerName The consumers name +- * @return string The path to file with PID +- */ +- private function getPidFilePath($consumerName) +- { +- return $consumerName . static::PID_FILE_EXT; ++ $connectionName = $consumerConfig->getConnection(); ++ try { ++ $this->mqConnectionTypeResolver->getConnectionType($connectionName); ++ } catch (\LogicException $e) { ++ $this->logger->info( ++ sprintf( ++ 'Consumer "%s" skipped as required connection "%s" is not configured. %s', ++ $consumerName, ++ $connectionName, ++ $e->getMessage() ++ ) ++ ); ++ return false; ++ } ++ ++ return true; + } + } +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php +deleted file mode 100644 +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php ++++ /dev/null +@@ -1,127 +0,0 @@ +-filesystem = $filesystem; +- } +- +- /** +- * Checks if consumer process is run by pid from pidFile +- * +- * @param string $pidFilePath The path to file with PID +- * @return bool Returns true if consumer process is run +- * @throws FileSystemException +- */ +- public function isRun($pidFilePath) +- { +- $pid = $this->getPid($pidFilePath); +- if ($pid) { +- if (function_exists('posix_getpgid')) { +- return (bool) posix_getpgid($pid); +- } else { +- return $this->checkIsProcessExists($pid); +- } +- } +- +- return false; +- } +- +- /** +- * Checks that process is running +- * +- * If php function exec is not available throws RuntimeException +- * If shell command returns non-zero code and this code is not 1 throws RuntimeException +- * +- * @param int $pid A pid of process +- * @return bool Returns true if consumer process is run +- * @throws \RuntimeException +- * @SuppressWarnings(PHPMD.UnusedLocalVariable) +- */ +- private function checkIsProcessExists($pid) +- { +- if (!function_exists('exec')) { +- throw new \RuntimeException('Function exec is not available'); +- } +- +- exec(escapeshellcmd('ps -p ' . $pid), $output, $code); +- +- $code = (int) $code; +- +- switch ($code) { +- case 0: +- return true; +- break; +- case 1: +- return false; +- break; +- default: +- throw new \RuntimeException('Exec returned non-zero code', $code); +- break; +- } +- } +- +- /** +- * Returns pid by pidFile path +- * +- * @param string $pidFilePath The path to file with PID +- * @return int Returns pid if pid file exists for consumer else returns 0 +- * @throws FileSystemException +- */ +- public function getPid($pidFilePath) +- { +- /** @var ReadInterface $directory */ +- $directory = $this->filesystem->getDirectoryRead(DirectoryList::VAR_DIR); +- +- if ($directory->isExist($pidFilePath)) { +- return (int) $directory->readFile($pidFilePath); +- } +- +- return 0; +- } +- +- /** +- * Saves pid of current process to file +- * +- * @param string $pidFilePath The path to file with pid +- * @throws FileSystemException +- */ +- public function savePid($pidFilePath) +- { +- /** @var WriteInterface $directory */ +- $directory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); +- $directory->writeFile($pidFilePath, function_exists('posix_getpid') ? posix_getpid() : getmypid(), 'w'); +- } +-} diff --git a/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.6.patch b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.6.patch new file mode 100644 index 0000000..a4b3d37 --- /dev/null +++ b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.6.patch @@ -0,0 +1,572 @@ +diff -Nuar a/vendor/magento/framework/Lock/Backend/Database.php b/vendor/magento/framework/Lock/Backend/Database.php +--- a/vendor/magento/framework/Lock/Backend/Database.php ++++ b/vendor/magento/framework/Lock/Backend/Database.php +@@ -3,8 +3,8 @@ + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +- + declare(strict_types=1); ++ + namespace Magento\Framework\Lock\Backend; + + use Magento\Framework\App\DeploymentConfig; +@@ -14,23 +14,40 @@ use Magento\Framework\Exception\AlreadyExistsException; + use Magento\Framework\Exception\InputException; + use Magento\Framework\Phrase; + ++/** ++ * Implementation of the lock manager on the basis of MySQL. ++ */ + class Database implements \Magento\Framework\Lock\LockManagerInterface + { +- /** @var ResourceConnection */ ++ /** ++ * Max time for lock is 1 week ++ * ++ * MariaDB does not support negative timeout value to get infinite timeout, ++ * so we set 1 week for lock timeout ++ */ ++ const MAX_LOCK_TIME = 604800; ++ ++ /** ++ * @var ResourceConnection ++ */ + private $resource; + +- /** @var DeploymentConfig */ ++ /** ++ * @var DeploymentConfig ++ */ + private $deploymentConfig; + +- /** @var string Lock prefix */ ++ /** ++ * @var string Lock prefix ++ */ + private $prefix; + +- /** @var string|false Holds current lock name if set, otherwise false */ ++ /** ++ * @var string|false Holds current lock name if set, otherwise false ++ */ + private $currentLock = false; + + /** +- * Database constructor. +- * + * @param ResourceConnection $resource + * @param DeploymentConfig $deploymentConfig + * @param string|null $prefix +@@ -53,9 +70,13 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @return bool + * @throws InputException + * @throws AlreadyExistsException ++ * @throws \Zend_Db_Statement_Exception + */ + public function lock(string $name, int $timeout = -1): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return true; ++ }; + $name = $this->addPrefix($name); + + /** +@@ -66,7 +87,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + if ($this->currentLock) { + throw new AlreadyExistsException( + new Phrase( +- 'Current connection is already holding lock for $1, only single lock allowed', ++ 'Current connection is already holding lock for %1, only single lock allowed', + [$this->currentLock] + ) + ); +@@ -74,7 +95,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + + $result = (bool)$this->resource->getConnection()->query( + "SELECT GET_LOCK(?, ?);", +- [(string)$name, (int)$timeout] ++ [$name, $timeout < 0 ? self::MAX_LOCK_TIME : $timeout] + )->fetchColumn(); + + if ($result === true) { +@@ -90,9 +111,14 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @param string $name lock name + * @return bool + * @throws InputException ++ * @throws \Zend_Db_Statement_Exception + */ + public function unlock(string $name): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return true; ++ }; ++ + $name = $this->addPrefix($name); + + $result = (bool)$this->resource->getConnection()->query( +@@ -113,14 +139,19 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @param string $name lock name + * @return bool + * @throws InputException ++ * @throws \Zend_Db_Statement_Exception + */ + public function isLocked(string $name): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return false; ++ }; ++ + $name = $this->addPrefix($name); + + return (bool)$this->resource->getConnection()->query( + "SELECT IS_USED_LOCK(?);", +- [(string)$name] ++ [$name] + )->fetchColumn(); + } + +@@ -130,7 +161,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * Limited to 64 characters in MySQL. + * + * @param string $name +- * @return string $name ++ * @return string + * @throws InputException + */ + private function addPrefix(string $name): string +diff -Nuar a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +--- a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php ++++ b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +@@ -11,7 +11,7 @@ use Symfony\Component\Console\Input\InputInterface; + use Symfony\Component\Console\Input\InputOption; + use Symfony\Component\Console\Output\OutputInterface; + use Magento\Framework\MessageQueue\ConsumerFactory; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Command for starting MessageQueue consumers. +@@ -22,6 +22,7 @@ class StartConsumerCommand extends Command + const OPTION_NUMBER_OF_MESSAGES = 'max-messages'; + const OPTION_BATCH_SIZE = 'batch-size'; + const OPTION_AREACODE = 'area-code'; ++ const OPTION_SINGLE_THREAD = 'single-thread'; + const PID_FILE_PATH = 'pid-file-path'; + const COMMAND_QUEUE_CONSUMERS_START = 'queue:consumers:start'; + +@@ -36,9 +37,9 @@ class StartConsumerCommand extends Command + private $appState; + + /** +- * @var PidConsumerManager ++ * @var LockManagerInterface + */ +- private $pidConsumerManager; ++ private $lockManager; + + /** + * StartConsumerCommand constructor. +@@ -47,23 +48,23 @@ class StartConsumerCommand extends Command + * @param \Magento\Framework\App\State $appState + * @param ConsumerFactory $consumerFactory + * @param string $name +- * @param PidConsumerManager $pidConsumerManager ++ * @param LockManagerInterface $lockManager + */ + public function __construct( + \Magento\Framework\App\State $appState, + ConsumerFactory $consumerFactory, + $name = null, +- PidConsumerManager $pidConsumerManager = null ++ LockManagerInterface $lockManager = null + ) { + $this->appState = $appState; + $this->consumerFactory = $consumerFactory; +- $this->pidConsumerManager = $pidConsumerManager ?: \Magento\Framework\App\ObjectManager::getInstance() +- ->get(PidConsumerManager::class); ++ $this->lockManager = $lockManager ?: \Magento\Framework\App\ObjectManager::getInstance() ++ ->get(LockManagerInterface::class); + parent::__construct($name); + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function execute(InputInterface $input, OutputInterface $output) + { +@@ -71,30 +72,36 @@ class StartConsumerCommand extends Command + $numberOfMessages = $input->getOption(self::OPTION_NUMBER_OF_MESSAGES); + $batchSize = (int)$input->getOption(self::OPTION_BATCH_SIZE); + $areaCode = $input->getOption(self::OPTION_AREACODE); +- $pidFilePath = $input->getOption(self::PID_FILE_PATH); + +- if ($pidFilePath && $this->pidConsumerManager->isRun($pidFilePath)) { +- $output->writeln('Consumer with the same PID is running'); +- return \Magento\Framework\Console\Cli::RETURN_FAILURE; ++ if ($input->getOption(self::PID_FILE_PATH)) { ++ $input->setOption(self::OPTION_SINGLE_THREAD, true); + } + +- if ($pidFilePath) { +- $this->pidConsumerManager->savePid($pidFilePath); ++ $singleThread = $input->getOption(self::OPTION_SINGLE_THREAD); ++ ++ if ($singleThread && $this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore ++ $output->writeln('Consumer with the same name is running'); ++ return \Magento\Framework\Console\Cli::RETURN_FAILURE; + } + +- if ($areaCode !== null) { +- $this->appState->setAreaCode($areaCode); +- } else { +- $this->appState->setAreaCode('global'); ++ if ($singleThread) { ++ $this->lockManager->lock(md5($consumerName)); //phpcs:ignore + } + ++ $this->appState->setAreaCode($areaCode ?? 'global'); ++ + $consumer = $this->consumerFactory->get($consumerName, $batchSize); + $consumer->process($numberOfMessages); ++ ++ if ($singleThread) { ++ $this->lockManager->unlock(md5($consumerName)); //phpcs:ignore ++ } ++ + return \Magento\Framework\Console\Cli::RETURN_SUCCESS; + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function configure() + { +@@ -125,11 +132,17 @@ class StartConsumerCommand extends Command + 'The preferred area (global, adminhtml, etc...) ' + . 'default is global.' + ); ++ $this->addOption( ++ self::OPTION_SINGLE_THREAD, ++ null, ++ InputOption::VALUE_NONE, ++ 'This option prevents running multiple copies of one consumer simultaneously.' ++ ); + $this->addOption( + self::PID_FILE_PATH, + null, + InputOption::VALUE_REQUIRED, +- 'The file path for saving PID' ++ 'The file path for saving PID (This option is deprecated, use --single-thread instead)' + ); + $this->setHelp( + <<%command.full_name% someConsumer --area-code='adminhtml' ++ ++To do not run multiple copies of one consumer simultaneously: ++ ++ %command.full_name% someConsumer --single-thread' + +-To save PID enter path: ++To save PID enter path (This option is deprecated, use --single-thread instead): + + %command.full_name% someConsumer --pid-file-path='/var/someConsumer.pid' + HELP +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php ++++ b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +@@ -5,22 +5,21 @@ + */ + namespace Magento\MessageQueue\Model\Cron; + ++use Magento\Framework\App\ObjectManager; ++use Magento\Framework\MessageQueue\ConnectionTypeResolver; ++use Magento\Framework\MessageQueue\Consumer\Config\ConsumerConfigItemInterface; + use Magento\Framework\ShellInterface; + use Magento\Framework\MessageQueue\Consumer\ConfigInterface as ConsumerConfigInterface; + use Magento\Framework\App\DeploymentConfig; ++use Psr\Log\LoggerInterface; + use Symfony\Component\Process\PhpExecutableFinder; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Class for running consumers processes by cron + */ + class ConsumersRunner + { +- /** +- * Extension of PID file +- */ +- const PID_FILE_EXT = '.pid'; +- + /** + * Shell command line wrapper for executing command in background + * +@@ -50,11 +49,21 @@ class ConsumersRunner + private $phpExecutableFinder; + + /** +- * The class for checking status of process by PID ++ * @var ConnectionTypeResolver ++ */ ++ private $mqConnectionTypeResolver; ++ ++ /** ++ * @var LoggerInterface ++ */ ++ private $logger; ++ ++ /** ++ * Lock Manager + * +- * @var PidConsumerManager ++ * @var LockManagerInterface + */ +- private $pidConsumerManager; ++ private $lockManager; + + /** + * @param PhpExecutableFinder $phpExecutableFinder The executable finder specifically designed +@@ -62,20 +71,28 @@ class ConsumersRunner + * @param ConsumerConfigInterface $consumerConfig The consumer config provider + * @param DeploymentConfig $deploymentConfig The application deployment configuration + * @param ShellInterface $shellBackground The shell command line wrapper for executing command in background +- * @param PidConsumerManager $pidConsumerManager The class for checking status of process by PID ++ * @param LockManagerInterface $lockManager The lock manager ++ * @param ConnectionTypeResolver $mqConnectionTypeResolver Consumer connection resolver ++ * @param LoggerInterface $logger Logger + */ + public function __construct( + PhpExecutableFinder $phpExecutableFinder, + ConsumerConfigInterface $consumerConfig, + DeploymentConfig $deploymentConfig, + ShellInterface $shellBackground, +- PidConsumerManager $pidConsumerManager ++ LockManagerInterface $lockManager, ++ ConnectionTypeResolver $mqConnectionTypeResolver = null, ++ LoggerInterface $logger = null + ) { + $this->phpExecutableFinder = $phpExecutableFinder; + $this->consumerConfig = $consumerConfig; + $this->deploymentConfig = $deploymentConfig; + $this->shellBackground = $shellBackground; +- $this->pidConsumerManager = $pidConsumerManager; ++ $this->lockManager = $lockManager; ++ $this->mqConnectionTypeResolver = $mqConnectionTypeResolver ++ ?: ObjectManager::getInstance()->get(ConnectionTypeResolver::class); ++ $this->logger = $logger ++ ?: ObjectManager::getInstance()->get(LoggerInterface::class); + } + + /** +@@ -94,15 +111,13 @@ class ConsumersRunner + $php = $this->phpExecutableFinder->find() ?: 'php'; + + foreach ($this->consumerConfig->getConsumers() as $consumer) { +- $consumerName = $consumer->getName(); +- +- if (!$this->canBeRun($consumerName, $allowedConsumers)) { ++ if (!$this->canBeRun($consumer, $allowedConsumers)) { + continue; + } + + $arguments = [ +- $consumerName, +- '--pid-file-path=' . $this->getPidFilePath($consumerName), ++ $consumer->getName(), ++ '--single-thread' + ]; + + if ($maxMessages) { +@@ -119,28 +134,38 @@ class ConsumersRunner + /** + * Checks that the consumer can be run + * +- * @param string $consumerName The consumer name ++ * @param ConsumerConfigItemInterface $consumerConfig The consumer config + * @param array $allowedConsumers The list of allowed consumers + * If $allowedConsumers is empty it means that all consumers are allowed + * @return bool Returns true if the consumer can be run ++ * @throws \Magento\Framework\Exception\FileSystemException + */ +- private function canBeRun($consumerName, array $allowedConsumers = []) ++ private function canBeRun(ConsumerConfigItemInterface $consumerConfig, array $allowedConsumers = []): bool + { +- $allowed = empty($allowedConsumers) ?: in_array($consumerName, $allowedConsumers); ++ $consumerName = $consumerConfig->getName(); ++ if (!empty($allowedConsumers) && !in_array($consumerName, $allowedConsumers)) { ++ return false; ++ } + +- return $allowed && !$this->pidConsumerManager->isRun($this->getPidFilePath($consumerName)); +- } ++ if ($this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore ++ return false; ++ } + +- /** +- * Returns default path to file with PID by consumers name +- * +- * @param string $consumerName The consumers name +- * @return string The path to file with PID +- */ +- private function getPidFilePath($consumerName) +- { +- $sanitizedHostname = preg_replace('/[^a-z0-9]/i', '', gethostname()); ++ $connectionName = $consumerConfig->getConnection(); ++ try { ++ $this->mqConnectionTypeResolver->getConnectionType($connectionName); ++ } catch (\LogicException $e) { ++ $this->logger->info( ++ sprintf( ++ 'Consumer "%s" skipped as required connection "%s" is not configured. %s', ++ $consumerName, ++ $connectionName, ++ $e->getMessage() ++ ) ++ ); ++ return false; ++ } + +- return $consumerName . '-' . $sanitizedHostname . static::PID_FILE_EXT; ++ return true; + } + } +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php +deleted file mode 100644 +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php ++++ /dev/null +@@ -1,127 +0,0 @@ +-filesystem = $filesystem; +- } +- +- /** +- * Checks if consumer process is run by pid from pidFile +- * +- * @param string $pidFilePath The path to file with PID +- * @return bool Returns true if consumer process is run +- * @throws FileSystemException +- */ +- public function isRun($pidFilePath) +- { +- $pid = $this->getPid($pidFilePath); +- if ($pid) { +- if (function_exists('posix_getpgid')) { +- return (bool) posix_getpgid($pid); +- } else { +- return $this->checkIsProcessExists($pid); +- } +- } +- +- return false; +- } +- +- /** +- * Checks that process is running +- * +- * If php function exec is not available throws RuntimeException +- * If shell command returns non-zero code and this code is not 1 throws RuntimeException +- * +- * @param int $pid A pid of process +- * @return bool Returns true if consumer process is run +- * @throws \RuntimeException +- * @SuppressWarnings(PHPMD.UnusedLocalVariable) +- */ +- private function checkIsProcessExists($pid) +- { +- if (!function_exists('exec')) { +- throw new \RuntimeException('Function exec is not available'); +- } +- +- exec(escapeshellcmd('ps -p ' . $pid), $output, $code); +- +- $code = (int) $code; +- +- switch ($code) { +- case 0: +- return true; +- break; +- case 1: +- return false; +- break; +- default: +- throw new \RuntimeException('Exec returned non-zero code', $code); +- break; +- } +- } +- +- /** +- * Returns pid by pidFile path +- * +- * @param string $pidFilePath The path to file with PID +- * @return int Returns pid if pid file exists for consumer else returns 0 +- * @throws FileSystemException +- */ +- public function getPid($pidFilePath) +- { +- /** @var ReadInterface $directory */ +- $directory = $this->filesystem->getDirectoryRead(DirectoryList::VAR_DIR); +- +- if ($directory->isExist($pidFilePath)) { +- return (int) $directory->readFile($pidFilePath); +- } +- +- return 0; +- } +- +- /** +- * Saves pid of current process to file +- * +- * @param string $pidFilePath The path to file with pid +- * @throws FileSystemException +- */ +- public function savePid($pidFilePath) +- { +- /** @var WriteInterface $directory */ +- $directory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); +- $directory->writeFile($pidFilePath, function_exists('posix_getpid') ? posix_getpid() : getmypid(), 'w'); +- } +-} diff --git a/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.7.patch b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.7.patch new file mode 100644 index 0000000..f7beda4 --- /dev/null +++ b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.7.patch @@ -0,0 +1,548 @@ +diff -Nuar a/vendor/magento/framework/Lock/Backend/Database.php b/vendor/magento/framework/Lock/Backend/Database.php +--- a/vendor/magento/framework/Lock/Backend/Database.php ++++ b/vendor/magento/framework/Lock/Backend/Database.php +@@ -3,8 +3,8 @@ + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +- + declare(strict_types=1); ++ + namespace Magento\Framework\Lock\Backend; + + use Magento\Framework\App\DeploymentConfig; +@@ -14,23 +14,40 @@ use Magento\Framework\Exception\AlreadyExistsException; + use Magento\Framework\Exception\InputException; + use Magento\Framework\Phrase; + ++/** ++ * Implementation of the lock manager on the basis of MySQL. ++ */ + class Database implements \Magento\Framework\Lock\LockManagerInterface + { +- /** @var ResourceConnection */ ++ /** ++ * Max time for lock is 1 week ++ * ++ * MariaDB does not support negative timeout value to get infinite timeout, ++ * so we set 1 week for lock timeout ++ */ ++ const MAX_LOCK_TIME = 604800; ++ ++ /** ++ * @var ResourceConnection ++ */ + private $resource; + +- /** @var DeploymentConfig */ ++ /** ++ * @var DeploymentConfig ++ */ + private $deploymentConfig; + +- /** @var string Lock prefix */ ++ /** ++ * @var string Lock prefix ++ */ + private $prefix; + +- /** @var string|false Holds current lock name if set, otherwise false */ ++ /** ++ * @var string|false Holds current lock name if set, otherwise false ++ */ + private $currentLock = false; + + /** +- * Database constructor. +- * + * @param ResourceConnection $resource + * @param DeploymentConfig $deploymentConfig + * @param string|null $prefix +@@ -53,9 +70,13 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @return bool + * @throws InputException + * @throws AlreadyExistsException ++ * @throws \Zend_Db_Statement_Exception + */ + public function lock(string $name, int $timeout = -1): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return true; ++ }; + $name = $this->addPrefix($name); + + /** +@@ -66,7 +87,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + if ($this->currentLock) { + throw new AlreadyExistsException( + new Phrase( +- 'Current connection is already holding lock for $1, only single lock allowed', ++ 'Current connection is already holding lock for %1, only single lock allowed', + [$this->currentLock] + ) + ); +@@ -74,7 +95,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + + $result = (bool)$this->resource->getConnection()->query( + "SELECT GET_LOCK(?, ?);", +- [(string)$name, (int)$timeout] ++ [$name, $timeout < 0 ? self::MAX_LOCK_TIME : $timeout] + )->fetchColumn(); + + if ($result === true) { +@@ -90,9 +111,14 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @param string $name lock name + * @return bool + * @throws InputException ++ * @throws \Zend_Db_Statement_Exception + */ + public function unlock(string $name): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return true; ++ }; ++ + $name = $this->addPrefix($name); + + $result = (bool)$this->resource->getConnection()->query( +@@ -113,14 +139,19 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @param string $name lock name + * @return bool + * @throws InputException ++ * @throws \Zend_Db_Statement_Exception + */ + public function isLocked(string $name): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return false; ++ }; ++ + $name = $this->addPrefix($name); + + return (bool)$this->resource->getConnection()->query( + "SELECT IS_USED_LOCK(?);", +- [(string)$name] ++ [$name] + )->fetchColumn(); + } + +@@ -130,7 +161,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * Limited to 64 characters in MySQL. + * + * @param string $name +- * @return string $name ++ * @return string + * @throws InputException + */ + private function addPrefix(string $name): string +diff -Nuar a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +--- a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php ++++ b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +@@ -11,7 +11,7 @@ use Symfony\Component\Console\Input\InputInterface; + use Symfony\Component\Console\Input\InputOption; + use Symfony\Component\Console\Output\OutputInterface; + use Magento\Framework\MessageQueue\ConsumerFactory; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Command for starting MessageQueue consumers. +@@ -22,6 +22,7 @@ class StartConsumerCommand extends Command + const OPTION_NUMBER_OF_MESSAGES = 'max-messages'; + const OPTION_BATCH_SIZE = 'batch-size'; + const OPTION_AREACODE = 'area-code'; ++ const OPTION_SINGLE_THREAD = 'single-thread'; + const PID_FILE_PATH = 'pid-file-path'; + const COMMAND_QUEUE_CONSUMERS_START = 'queue:consumers:start'; + +@@ -36,9 +37,9 @@ class StartConsumerCommand extends Command + private $appState; + + /** +- * @var PidConsumerManager ++ * @var LockManagerInterface + */ +- private $pidConsumerManager; ++ private $lockManager; + + /** + * StartConsumerCommand constructor. +@@ -47,23 +48,23 @@ class StartConsumerCommand extends Command + * @param \Magento\Framework\App\State $appState + * @param ConsumerFactory $consumerFactory + * @param string $name +- * @param PidConsumerManager $pidConsumerManager ++ * @param LockManagerInterface $lockManager + */ + public function __construct( + \Magento\Framework\App\State $appState, + ConsumerFactory $consumerFactory, + $name = null, +- PidConsumerManager $pidConsumerManager = null ++ LockManagerInterface $lockManager = null + ) { + $this->appState = $appState; + $this->consumerFactory = $consumerFactory; +- $this->pidConsumerManager = $pidConsumerManager ?: \Magento\Framework\App\ObjectManager::getInstance() +- ->get(PidConsumerManager::class); ++ $this->lockManager = $lockManager ?: \Magento\Framework\App\ObjectManager::getInstance() ++ ->get(LockManagerInterface::class); + parent::__construct($name); + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function execute(InputInterface $input, OutputInterface $output) + { +@@ -71,30 +72,36 @@ class StartConsumerCommand extends Command + $numberOfMessages = $input->getOption(self::OPTION_NUMBER_OF_MESSAGES); + $batchSize = (int)$input->getOption(self::OPTION_BATCH_SIZE); + $areaCode = $input->getOption(self::OPTION_AREACODE); +- $pidFilePath = $input->getOption(self::PID_FILE_PATH); + +- if ($pidFilePath && $this->pidConsumerManager->isRun($pidFilePath)) { +- $output->writeln('Consumer with the same PID is running'); +- return \Magento\Framework\Console\Cli::RETURN_FAILURE; ++ if ($input->getOption(self::PID_FILE_PATH)) { ++ $input->setOption(self::OPTION_SINGLE_THREAD, true); + } + +- if ($pidFilePath) { +- $this->pidConsumerManager->savePid($pidFilePath); ++ $singleThread = $input->getOption(self::OPTION_SINGLE_THREAD); ++ ++ if ($singleThread && $this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore ++ $output->writeln('Consumer with the same name is running'); ++ return \Magento\Framework\Console\Cli::RETURN_FAILURE; + } + +- if ($areaCode !== null) { +- $this->appState->setAreaCode($areaCode); +- } else { +- $this->appState->setAreaCode('global'); ++ if ($singleThread) { ++ $this->lockManager->lock(md5($consumerName)); //phpcs:ignore + } + ++ $this->appState->setAreaCode($areaCode ?? 'global'); ++ + $consumer = $this->consumerFactory->get($consumerName, $batchSize); + $consumer->process($numberOfMessages); ++ ++ if ($singleThread) { ++ $this->lockManager->unlock(md5($consumerName)); //phpcs:ignore ++ } ++ + return \Magento\Framework\Console\Cli::RETURN_SUCCESS; + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function configure() + { +@@ -125,11 +132,17 @@ class StartConsumerCommand extends Command + 'The preferred area (global, adminhtml, etc...) ' + . 'default is global.' + ); ++ $this->addOption( ++ self::OPTION_SINGLE_THREAD, ++ null, ++ InputOption::VALUE_NONE, ++ 'This option prevents running multiple copies of one consumer simultaneously.' ++ ); + $this->addOption( + self::PID_FILE_PATH, + null, + InputOption::VALUE_REQUIRED, +- 'The file path for saving PID' ++ 'The file path for saving PID (This option is deprecated, use --single-thread instead)' + ); + $this->setHelp( + <<%command.full_name% someConsumer --area-code='adminhtml' ++ ++To do not run multiple copies of one consumer simultaneously: ++ ++ %command.full_name% someConsumer --single-thread' + +-To save PID enter path: ++To save PID enter path (This option is deprecated, use --single-thread instead): + + %command.full_name% someConsumer --pid-file-path='/var/someConsumer.pid' + HELP +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php ++++ b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +@@ -13,18 +13,13 @@ use Magento\Framework\MessageQueue\Consumer\ConfigInterface as ConsumerConfigInt + use Magento\Framework\App\DeploymentConfig; + use Psr\Log\LoggerInterface; + use Symfony\Component\Process\PhpExecutableFinder; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Class for running consumers processes by cron + */ + class ConsumersRunner + { +- /** +- * Extension of PID file +- */ +- const PID_FILE_EXT = '.pid'; +- + /** + * Shell command line wrapper for executing command in background + * +@@ -53,13 +48,6 @@ class ConsumersRunner + */ + private $phpExecutableFinder; + +- /** +- * The class for checking status of process by PID +- * +- * @var PidConsumerManager +- */ +- private $pidConsumerManager; +- + /** + * @var ConnectionTypeResolver + */ +@@ -70,13 +58,20 @@ class ConsumersRunner + */ + private $logger; + ++ /** ++ * Lock Manager ++ * ++ * @var LockManagerInterface ++ */ ++ private $lockManager; ++ + /** + * @param PhpExecutableFinder $phpExecutableFinder The executable finder specifically designed + * for the PHP executable + * @param ConsumerConfigInterface $consumerConfig The consumer config provider + * @param DeploymentConfig $deploymentConfig The application deployment configuration + * @param ShellInterface $shellBackground The shell command line wrapper for executing command in background +- * @param PidConsumerManager $pidConsumerManager The class for checking status of process by PID ++ * @param LockManagerInterface $lockManager The lock manager + * @param ConnectionTypeResolver $mqConnectionTypeResolver Consumer connection resolver + * @param LoggerInterface $logger Logger + */ +@@ -85,7 +80,7 @@ class ConsumersRunner + ConsumerConfigInterface $consumerConfig, + DeploymentConfig $deploymentConfig, + ShellInterface $shellBackground, +- PidConsumerManager $pidConsumerManager, ++ LockManagerInterface $lockManager, + ConnectionTypeResolver $mqConnectionTypeResolver = null, + LoggerInterface $logger = null + ) { +@@ -93,7 +88,7 @@ class ConsumersRunner + $this->consumerConfig = $consumerConfig; + $this->deploymentConfig = $deploymentConfig; + $this->shellBackground = $shellBackground; +- $this->pidConsumerManager = $pidConsumerManager; ++ $this->lockManager = $lockManager; + $this->mqConnectionTypeResolver = $mqConnectionTypeResolver + ?: ObjectManager::getInstance()->get(ConnectionTypeResolver::class); + $this->logger = $logger +@@ -120,11 +115,9 @@ class ConsumersRunner + continue; + } + +- $consumerName = $consumer->getName(); +- + $arguments = [ +- $consumerName, +- '--pid-file-path=' . $this->getPidFilePath($consumerName), ++ $consumer->getName(), ++ '--single-thread' + ]; + + if ($maxMessages) { +@@ -154,7 +147,7 @@ class ConsumersRunner + return false; + } + +- if ($this->pidConsumerManager->isRun($this->getPidFilePath($consumerName))) { ++ if ($this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore + return false; + } + +@@ -162,28 +155,17 @@ class ConsumersRunner + try { + $this->mqConnectionTypeResolver->getConnectionType($connectionName); + } catch (\LogicException $e) { +- $this->logger->info(sprintf( +- 'Consumer "%s" skipped as required connection "%s" is not configured. %s', +- $consumerName, +- $connectionName, +- $e->getMessage() +- )); ++ $this->logger->info( ++ sprintf( ++ 'Consumer "%s" skipped as required connection "%s" is not configured. %s', ++ $consumerName, ++ $connectionName, ++ $e->getMessage() ++ ) ++ ); + return false; + } + + return true; + } +- +- /** +- * Returns default path to file with PID by consumers name +- * +- * @param string $consumerName The consumers name +- * @return string The path to file with PID +- */ +- private function getPidFilePath($consumerName) +- { +- $sanitizedHostname = preg_replace('/[^a-z0-9]/i', '', gethostname()); +- +- return $consumerName . '-' . $sanitizedHostname . static::PID_FILE_EXT; +- } + } +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php +deleted file mode 100644 +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php ++++ /dev/null +@@ -1,127 +0,0 @@ +-filesystem = $filesystem; +- } +- +- /** +- * Checks if consumer process is run by pid from pidFile +- * +- * @param string $pidFilePath The path to file with PID +- * @return bool Returns true if consumer process is run +- * @throws FileSystemException +- */ +- public function isRun($pidFilePath) +- { +- $pid = $this->getPid($pidFilePath); +- if ($pid) { +- if (function_exists('posix_getpgid')) { +- return (bool) posix_getpgid($pid); +- } else { +- return $this->checkIsProcessExists($pid); +- } +- } +- +- return false; +- } +- +- /** +- * Checks that process is running +- * +- * If php function exec is not available throws RuntimeException +- * If shell command returns non-zero code and this code is not 1 throws RuntimeException +- * +- * @param int $pid A pid of process +- * @return bool Returns true if consumer process is run +- * @throws \RuntimeException +- * @SuppressWarnings(PHPMD.UnusedLocalVariable) +- */ +- private function checkIsProcessExists($pid) +- { +- if (!function_exists('exec')) { +- throw new \RuntimeException('Function exec is not available'); +- } +- +- exec(escapeshellcmd('ps -p ' . $pid), $output, $code); +- +- $code = (int) $code; +- +- switch ($code) { +- case 0: +- return true; +- break; +- case 1: +- return false; +- break; +- default: +- throw new \RuntimeException('Exec returned non-zero code', $code); +- break; +- } +- } +- +- /** +- * Returns pid by pidFile path +- * +- * @param string $pidFilePath The path to file with PID +- * @return int Returns pid if pid file exists for consumer else returns 0 +- * @throws FileSystemException +- */ +- public function getPid($pidFilePath) +- { +- /** @var ReadInterface $directory */ +- $directory = $this->filesystem->getDirectoryRead(DirectoryList::VAR_DIR); +- +- if ($directory->isExist($pidFilePath)) { +- return (int) $directory->readFile($pidFilePath); +- } +- +- return 0; +- } +- +- /** +- * Saves pid of current process to file +- * +- * @param string $pidFilePath The path to file with pid +- * @throws FileSystemException +- */ +- public function savePid($pidFilePath) +- { +- /** @var WriteInterface $directory */ +- $directory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); +- $directory->writeFile($pidFilePath, function_exists('posix_getpid') ? posix_getpid() : getmypid(), 'w'); +- } +-} diff --git a/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.8.patch b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.8.patch new file mode 100644 index 0000000..9be3c50 --- /dev/null +++ b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.8.patch @@ -0,0 +1,496 @@ +diff -Nuar a/vendor/magento/framework/Lock/Backend/Database.php b/vendor/magento/framework/Lock/Backend/Database.php +--- a/vendor/magento/framework/Lock/Backend/Database.php ++++ b/vendor/magento/framework/Lock/Backend/Database.php +@@ -19,21 +19,35 @@ use Magento\Framework\Phrase; + */ + class Database implements \Magento\Framework\Lock\LockManagerInterface + { +- /** @var ResourceConnection */ ++ /** ++ * Max time for lock is 1 week ++ * ++ * MariaDB does not support negative timeout value to get infinite timeout, ++ * so we set 1 week for lock timeout ++ */ ++ const MAX_LOCK_TIME = 604800; ++ ++ /** ++ * @var ResourceConnection ++ */ + private $resource; + +- /** @var DeploymentConfig */ ++ /** ++ * @var DeploymentConfig ++ */ + private $deploymentConfig; + +- /** @var string Lock prefix */ ++ /** ++ * @var string Lock prefix ++ */ + private $prefix; + +- /** @var string|false Holds current lock name if set, otherwise false */ ++ /** ++ * @var string|false Holds current lock name if set, otherwise false ++ */ + private $currentLock = false; + + /** +- * Database constructor. +- * + * @param ResourceConnection $resource + * @param DeploymentConfig $deploymentConfig + * @param string|null $prefix +@@ -81,7 +95,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + + $result = (bool)$this->resource->getConnection()->query( + "SELECT GET_LOCK(?, ?);", +- [(string)$name, (int)$timeout] ++ [$name, $timeout < 0 ? self::MAX_LOCK_TIME : $timeout] + )->fetchColumn(); + + if ($result === true) { +@@ -104,6 +118,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + if (!$this->deploymentConfig->isDbAvailable()) { + return true; + }; ++ + $name = $this->addPrefix($name); + + $result = (bool)$this->resource->getConnection()->query( +@@ -131,11 +146,12 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + if (!$this->deploymentConfig->isDbAvailable()) { + return false; + }; ++ + $name = $this->addPrefix($name); + + return (bool)$this->resource->getConnection()->query( + "SELECT IS_USED_LOCK(?);", +- [(string)$name] ++ [$name] + )->fetchColumn(); + } + +@@ -145,7 +161,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * Limited to 64 characters in MySQL. + * + * @param string $name +- * @return string $name ++ * @return string + * @throws InputException + */ + private function addPrefix(string $name): string +diff -Nuar a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +--- a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php ++++ b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +@@ -11,7 +11,7 @@ use Symfony\Component\Console\Input\InputInterface; + use Symfony\Component\Console\Input\InputOption; + use Symfony\Component\Console\Output\OutputInterface; + use Magento\Framework\MessageQueue\ConsumerFactory; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Command for starting MessageQueue consumers. +@@ -22,6 +22,7 @@ class StartConsumerCommand extends Command + const OPTION_NUMBER_OF_MESSAGES = 'max-messages'; + const OPTION_BATCH_SIZE = 'batch-size'; + const OPTION_AREACODE = 'area-code'; ++ const OPTION_SINGLE_THREAD = 'single-thread'; + const PID_FILE_PATH = 'pid-file-path'; + const COMMAND_QUEUE_CONSUMERS_START = 'queue:consumers:start'; + +@@ -36,9 +37,9 @@ class StartConsumerCommand extends Command + private $appState; + + /** +- * @var PidConsumerManager ++ * @var LockManagerInterface + */ +- private $pidConsumerManager; ++ private $lockManager; + + /** + * StartConsumerCommand constructor. +@@ -47,23 +48,23 @@ class StartConsumerCommand extends Command + * @param \Magento\Framework\App\State $appState + * @param ConsumerFactory $consumerFactory + * @param string $name +- * @param PidConsumerManager $pidConsumerManager ++ * @param LockManagerInterface $lockManager + */ + public function __construct( + \Magento\Framework\App\State $appState, + ConsumerFactory $consumerFactory, + $name = null, +- PidConsumerManager $pidConsumerManager = null ++ LockManagerInterface $lockManager = null + ) { + $this->appState = $appState; + $this->consumerFactory = $consumerFactory; +- $this->pidConsumerManager = $pidConsumerManager ?: \Magento\Framework\App\ObjectManager::getInstance() +- ->get(PidConsumerManager::class); ++ $this->lockManager = $lockManager ?: \Magento\Framework\App\ObjectManager::getInstance() ++ ->get(LockManagerInterface::class); + parent::__construct($name); + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function execute(InputInterface $input, OutputInterface $output) + { +@@ -71,30 +72,36 @@ class StartConsumerCommand extends Command + $numberOfMessages = $input->getOption(self::OPTION_NUMBER_OF_MESSAGES); + $batchSize = (int)$input->getOption(self::OPTION_BATCH_SIZE); + $areaCode = $input->getOption(self::OPTION_AREACODE); +- $pidFilePath = $input->getOption(self::PID_FILE_PATH); + +- if ($pidFilePath && $this->pidConsumerManager->isRun($pidFilePath)) { +- $output->writeln('Consumer with the same PID is running'); +- return \Magento\Framework\Console\Cli::RETURN_FAILURE; ++ if ($input->getOption(self::PID_FILE_PATH)) { ++ $input->setOption(self::OPTION_SINGLE_THREAD, true); + } + +- if ($pidFilePath) { +- $this->pidConsumerManager->savePid($pidFilePath); ++ $singleThread = $input->getOption(self::OPTION_SINGLE_THREAD); ++ ++ if ($singleThread && $this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore ++ $output->writeln('Consumer with the same name is running'); ++ return \Magento\Framework\Console\Cli::RETURN_FAILURE; + } + +- if ($areaCode !== null) { +- $this->appState->setAreaCode($areaCode); +- } else { +- $this->appState->setAreaCode('global'); ++ if ($singleThread) { ++ $this->lockManager->lock(md5($consumerName)); //phpcs:ignore + } + ++ $this->appState->setAreaCode($areaCode ?? 'global'); ++ + $consumer = $this->consumerFactory->get($consumerName, $batchSize); + $consumer->process($numberOfMessages); ++ ++ if ($singleThread) { ++ $this->lockManager->unlock(md5($consumerName)); //phpcs:ignore ++ } ++ + return \Magento\Framework\Console\Cli::RETURN_SUCCESS; + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function configure() + { +@@ -125,11 +132,17 @@ class StartConsumerCommand extends Command + 'The preferred area (global, adminhtml, etc...) ' + . 'default is global.' + ); ++ $this->addOption( ++ self::OPTION_SINGLE_THREAD, ++ null, ++ InputOption::VALUE_NONE, ++ 'This option prevents running multiple copies of one consumer simultaneously.' ++ ); + $this->addOption( + self::PID_FILE_PATH, + null, + InputOption::VALUE_REQUIRED, +- 'The file path for saving PID' ++ 'The file path for saving PID (This option is deprecated, use --single-thread instead)' + ); + $this->setHelp( + <<%command.full_name% someConsumer --area-code='adminhtml' ++ ++To do not run multiple copies of one consumer simultaneously: ++ ++ %command.full_name% someConsumer --single-thread' + +-To save PID enter path: ++To save PID enter path (This option is deprecated, use --single-thread instead): + + %command.full_name% someConsumer --pid-file-path='/var/someConsumer.pid' + HELP +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php ++++ b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +@@ -13,18 +13,13 @@ use Magento\Framework\MessageQueue\Consumer\ConfigInterface as ConsumerConfigInt + use Magento\Framework\App\DeploymentConfig; + use Psr\Log\LoggerInterface; + use Symfony\Component\Process\PhpExecutableFinder; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Class for running consumers processes by cron + */ + class ConsumersRunner + { +- /** +- * Extension of PID file +- */ +- const PID_FILE_EXT = '.pid'; +- + /** + * Shell command line wrapper for executing command in background + * +@@ -53,13 +48,6 @@ class ConsumersRunner + */ + private $phpExecutableFinder; + +- /** +- * The class for checking status of process by PID +- * +- * @var PidConsumerManager +- */ +- private $pidConsumerManager; +- + /** + * @var ConnectionTypeResolver + */ +@@ -70,13 +58,20 @@ class ConsumersRunner + */ + private $logger; + ++ /** ++ * Lock Manager ++ * ++ * @var LockManagerInterface ++ */ ++ private $lockManager; ++ + /** + * @param PhpExecutableFinder $phpExecutableFinder The executable finder specifically designed + * for the PHP executable + * @param ConsumerConfigInterface $consumerConfig The consumer config provider + * @param DeploymentConfig $deploymentConfig The application deployment configuration + * @param ShellInterface $shellBackground The shell command line wrapper for executing command in background +- * @param PidConsumerManager $pidConsumerManager The class for checking status of process by PID ++ * @param LockManagerInterface $lockManager The lock manager + * @param ConnectionTypeResolver $mqConnectionTypeResolver Consumer connection resolver + * @param LoggerInterface $logger Logger + */ +@@ -85,7 +80,7 @@ class ConsumersRunner + ConsumerConfigInterface $consumerConfig, + DeploymentConfig $deploymentConfig, + ShellInterface $shellBackground, +- PidConsumerManager $pidConsumerManager, ++ LockManagerInterface $lockManager, + ConnectionTypeResolver $mqConnectionTypeResolver = null, + LoggerInterface $logger = null + ) { +@@ -93,7 +88,7 @@ class ConsumersRunner + $this->consumerConfig = $consumerConfig; + $this->deploymentConfig = $deploymentConfig; + $this->shellBackground = $shellBackground; +- $this->pidConsumerManager = $pidConsumerManager; ++ $this->lockManager = $lockManager; + $this->mqConnectionTypeResolver = $mqConnectionTypeResolver + ?: ObjectManager::getInstance()->get(ConnectionTypeResolver::class); + $this->logger = $logger +@@ -120,11 +115,9 @@ class ConsumersRunner + continue; + } + +- $consumerName = $consumer->getName(); +- + $arguments = [ +- $consumerName, +- '--pid-file-path=' . $this->getPidFilePath($consumerName), ++ $consumer->getName(), ++ '--single-thread' + ]; + + if ($maxMessages) { +@@ -154,7 +147,7 @@ class ConsumersRunner + return false; + } + +- if ($this->pidConsumerManager->isRun($this->getPidFilePath($consumerName))) { ++ if ($this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore + return false; + } + +@@ -162,28 +155,17 @@ class ConsumersRunner + try { + $this->mqConnectionTypeResolver->getConnectionType($connectionName); + } catch (\LogicException $e) { +- $this->logger->info(sprintf( +- 'Consumer "%s" skipped as required connection "%s" is not configured. %s', +- $consumerName, +- $connectionName, +- $e->getMessage() +- )); ++ $this->logger->info( ++ sprintf( ++ 'Consumer "%s" skipped as required connection "%s" is not configured. %s', ++ $consumerName, ++ $connectionName, ++ $e->getMessage() ++ ) ++ ); + return false; + } + + return true; + } +- +- /** +- * Returns default path to file with PID by consumers name +- * +- * @param string $consumerName The consumers name +- * @return string The path to file with PID +- */ +- private function getPidFilePath($consumerName) +- { +- $sanitizedHostname = preg_replace('/[^a-z0-9]/i', '', gethostname()); +- +- return $consumerName . '-' . $sanitizedHostname . static::PID_FILE_EXT; +- } + } +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php +deleted file mode 100644 +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php ++++ /dev/null +@@ -1,127 +0,0 @@ +-filesystem = $filesystem; +- } +- +- /** +- * Checks if consumer process is run by pid from pidFile +- * +- * @param string $pidFilePath The path to file with PID +- * @return bool Returns true if consumer process is run +- * @throws FileSystemException +- */ +- public function isRun($pidFilePath) +- { +- $pid = $this->getPid($pidFilePath); +- if ($pid) { +- if (function_exists('posix_getpgid')) { +- return (bool) posix_getpgid($pid); +- } else { +- return $this->checkIsProcessExists($pid); +- } +- } +- +- return false; +- } +- +- /** +- * Checks that process is running +- * +- * If php function exec is not available throws RuntimeException +- * If shell command returns non-zero code and this code is not 1 throws RuntimeException +- * +- * @param int $pid A pid of process +- * @return bool Returns true if consumer process is run +- * @throws \RuntimeException +- * @SuppressWarnings(PHPMD.UnusedLocalVariable) +- */ +- private function checkIsProcessExists($pid) +- { +- if (!function_exists('exec')) { +- throw new \RuntimeException('Function exec is not available'); +- } +- +- exec(escapeshellcmd('ps -p ' . $pid), $output, $code); +- +- $code = (int) $code; +- +- switch ($code) { +- case 0: +- return true; +- break; +- case 1: +- return false; +- break; +- default: +- throw new \RuntimeException('Exec returned non-zero code', $code); +- break; +- } +- } +- +- /** +- * Returns pid by pidFile path +- * +- * @param string $pidFilePath The path to file with PID +- * @return int Returns pid if pid file exists for consumer else returns 0 +- * @throws FileSystemException +- */ +- public function getPid($pidFilePath) +- { +- /** @var ReadInterface $directory */ +- $directory = $this->filesystem->getDirectoryRead(DirectoryList::VAR_DIR); +- +- if ($directory->isExist($pidFilePath)) { +- return (int) $directory->readFile($pidFilePath); +- } +- +- return 0; +- } +- +- /** +- * Saves pid of current process to file +- * +- * @param string $pidFilePath The path to file with pid +- * @throws FileSystemException +- */ +- public function savePid($pidFilePath) +- { +- /** @var WriteInterface $directory */ +- $directory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); +- $directory->writeFile($pidFilePath, function_exists('posix_getpid') ? posix_getpid() : getmypid(), 'w'); +- } +-} diff --git a/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.0.patch b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.0.patch new file mode 100644 index 0000000..d231ac6 --- /dev/null +++ b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.0.patch @@ -0,0 +1,511 @@ +diff -Nuar a/vendor/magento/framework/Lock/Backend/Database.php b/vendor/magento/framework/Lock/Backend/Database.php +--- a/vendor/magento/framework/Lock/Backend/Database.php ++++ b/vendor/magento/framework/Lock/Backend/Database.php +@@ -15,10 +15,18 @@ use Magento\Framework\Exception\InputException; + use Magento\Framework\Phrase; + + /** +- * LockManager using the DB locks ++ * Implementation of the lock manager on the basis of MySQL. + */ + class Database implements \Magento\Framework\Lock\LockManagerInterface + { ++ /** ++ * Max time for lock is 1 week ++ * ++ * MariaDB does not support negative timeout value to get infinite timeout, ++ * so we set 1 week for lock timeout ++ */ ++ private const MAX_LOCK_TIME = 604800; ++ + /** + * @var ResourceConnection + */ +@@ -62,9 +70,13 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @return bool + * @throws InputException + * @throws AlreadyExistsException ++ * @throws \Zend_Db_Statement_Exception + */ + public function lock(string $name, int $timeout = -1): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return true; ++ }; + $name = $this->addPrefix($name); + + /** +@@ -75,7 +87,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + if ($this->currentLock) { + throw new AlreadyExistsException( + new Phrase( +- 'Current connection is already holding lock for $1, only single lock allowed', ++ 'Current connection is already holding lock for %1, only single lock allowed', + [$this->currentLock] + ) + ); +@@ -83,7 +95,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + + $result = (bool)$this->resource->getConnection()->query( + "SELECT GET_LOCK(?, ?);", +- [(string)$name, (int)$timeout] ++ [$name, $timeout < 0 ? self::MAX_LOCK_TIME : $timeout] + )->fetchColumn(); + + if ($result === true) { +@@ -99,9 +111,14 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @param string $name lock name + * @return bool + * @throws InputException ++ * @throws \Zend_Db_Statement_Exception + */ + public function unlock(string $name): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return true; ++ }; ++ + $name = $this->addPrefix($name); + + $result = (bool)$this->resource->getConnection()->query( +@@ -122,14 +139,19 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * @param string $name lock name + * @return bool + * @throws InputException ++ * @throws \Zend_Db_Statement_Exception + */ + public function isLocked(string $name): bool + { ++ if (!$this->deploymentConfig->isDbAvailable()) { ++ return false; ++ }; ++ + $name = $this->addPrefix($name); + + return (bool)$this->resource->getConnection()->query( + "SELECT IS_USED_LOCK(?);", +- [(string)$name] ++ [$name] + )->fetchColumn(); + } + +@@ -139,7 +161,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + * Limited to 64 characters in MySQL. + * + * @param string $name +- * @return string $name ++ * @return string + * @throws InputException + */ + private function addPrefix(string $name): string +diff -Nuar a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +--- a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php ++++ b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +@@ -11,7 +11,7 @@ use Symfony\Component\Console\Input\InputInterface; + use Symfony\Component\Console\Input\InputOption; + use Symfony\Component\Console\Output\OutputInterface; + use Magento\Framework\MessageQueue\ConsumerFactory; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Command for starting MessageQueue consumers. +@@ -22,6 +22,7 @@ class StartConsumerCommand extends Command + const OPTION_NUMBER_OF_MESSAGES = 'max-messages'; + const OPTION_BATCH_SIZE = 'batch-size'; + const OPTION_AREACODE = 'area-code'; ++ const OPTION_SINGLE_THREAD = 'single-thread'; + const PID_FILE_PATH = 'pid-file-path'; + const COMMAND_QUEUE_CONSUMERS_START = 'queue:consumers:start'; + +@@ -36,9 +37,9 @@ class StartConsumerCommand extends Command + private $appState; + + /** +- * @var PidConsumerManager ++ * @var LockManagerInterface + */ +- private $pidConsumerManager; ++ private $lockManager; + + /** + * StartConsumerCommand constructor. +@@ -47,23 +48,23 @@ class StartConsumerCommand extends Command + * @param \Magento\Framework\App\State $appState + * @param ConsumerFactory $consumerFactory + * @param string $name +- * @param PidConsumerManager $pidConsumerManager ++ * @param LockManagerInterface $lockManager + */ + public function __construct( + \Magento\Framework\App\State $appState, + ConsumerFactory $consumerFactory, + $name = null, +- PidConsumerManager $pidConsumerManager = null ++ LockManagerInterface $lockManager = null + ) { + $this->appState = $appState; + $this->consumerFactory = $consumerFactory; +- $this->pidConsumerManager = $pidConsumerManager ?: \Magento\Framework\App\ObjectManager::getInstance() +- ->get(PidConsumerManager::class); ++ $this->lockManager = $lockManager ?: \Magento\Framework\App\ObjectManager::getInstance() ++ ->get(LockManagerInterface::class); + parent::__construct($name); + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function execute(InputInterface $input, OutputInterface $output) + { +@@ -71,30 +72,36 @@ class StartConsumerCommand extends Command + $numberOfMessages = $input->getOption(self::OPTION_NUMBER_OF_MESSAGES); + $batchSize = (int)$input->getOption(self::OPTION_BATCH_SIZE); + $areaCode = $input->getOption(self::OPTION_AREACODE); +- $pidFilePath = $input->getOption(self::PID_FILE_PATH); + +- if ($pidFilePath && $this->pidConsumerManager->isRun($pidFilePath)) { +- $output->writeln('Consumer with the same PID is running'); +- return \Magento\Framework\Console\Cli::RETURN_FAILURE; ++ if ($input->getOption(self::PID_FILE_PATH)) { ++ $input->setOption(self::OPTION_SINGLE_THREAD, true); + } + +- if ($pidFilePath) { +- $this->pidConsumerManager->savePid($pidFilePath); ++ $singleThread = $input->getOption(self::OPTION_SINGLE_THREAD); ++ ++ if ($singleThread && $this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore ++ $output->writeln('Consumer with the same name is running'); ++ return \Magento\Framework\Console\Cli::RETURN_FAILURE; + } + +- if ($areaCode !== null) { +- $this->appState->setAreaCode($areaCode); +- } else { +- $this->appState->setAreaCode('global'); ++ if ($singleThread) { ++ $this->lockManager->lock(md5($consumerName)); //phpcs:ignore + } + ++ $this->appState->setAreaCode($areaCode ?? 'global'); ++ + $consumer = $this->consumerFactory->get($consumerName, $batchSize); + $consumer->process($numberOfMessages); ++ ++ if ($singleThread) { ++ $this->lockManager->unlock(md5($consumerName)); //phpcs:ignore ++ } ++ + return \Magento\Framework\Console\Cli::RETURN_SUCCESS; + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function configure() + { +@@ -125,11 +132,17 @@ class StartConsumerCommand extends Command + 'The preferred area (global, adminhtml, etc...) ' + . 'default is global.' + ); ++ $this->addOption( ++ self::OPTION_SINGLE_THREAD, ++ null, ++ InputOption::VALUE_NONE, ++ 'This option prevents running multiple copies of one consumer simultaneously.' ++ ); + $this->addOption( + self::PID_FILE_PATH, + null, + InputOption::VALUE_REQUIRED, +- 'The file path for saving PID' ++ 'The file path for saving PID (This option is deprecated, use --single-thread instead)' + ); + $this->setHelp( + <<%command.full_name% someConsumer --area-code='adminhtml' ++ ++To do not run multiple copies of one consumer simultaneously: ++ ++ %command.full_name% someConsumer --single-thread' + +-To save PID enter path: ++To save PID enter path (This option is deprecated, use --single-thread instead): + + %command.full_name% someConsumer --pid-file-path='/var/someConsumer.pid' + HELP +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php ++++ b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +@@ -13,18 +13,13 @@ use Magento\Framework\MessageQueue\Consumer\ConfigInterface as ConsumerConfigInt + use Magento\Framework\App\DeploymentConfig; + use Psr\Log\LoggerInterface; + use Symfony\Component\Process\PhpExecutableFinder; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Class for running consumers processes by cron + */ + class ConsumersRunner + { +- /** +- * Extension of PID file +- */ +- const PID_FILE_EXT = '.pid'; +- + /** + * Shell command line wrapper for executing command in background + * +@@ -53,13 +48,6 @@ class ConsumersRunner + */ + private $phpExecutableFinder; + +- /** +- * The class for checking status of process by PID +- * +- * @var PidConsumerManager +- */ +- private $pidConsumerManager; +- + /** + * @var ConnectionTypeResolver + */ +@@ -70,13 +58,20 @@ class ConsumersRunner + */ + private $logger; + ++ /** ++ * Lock Manager ++ * ++ * @var LockManagerInterface ++ */ ++ private $lockManager; ++ + /** + * @param PhpExecutableFinder $phpExecutableFinder The executable finder specifically designed + * for the PHP executable + * @param ConsumerConfigInterface $consumerConfig The consumer config provider + * @param DeploymentConfig $deploymentConfig The application deployment configuration + * @param ShellInterface $shellBackground The shell command line wrapper for executing command in background +- * @param PidConsumerManager $pidConsumerManager The class for checking status of process by PID ++ * @param LockManagerInterface $lockManager The lock manager + * @param ConnectionTypeResolver $mqConnectionTypeResolver Consumer connection resolver + * @param LoggerInterface $logger Logger + */ +@@ -85,7 +80,7 @@ class ConsumersRunner + ConsumerConfigInterface $consumerConfig, + DeploymentConfig $deploymentConfig, + ShellInterface $shellBackground, +- PidConsumerManager $pidConsumerManager, ++ LockManagerInterface $lockManager, + ConnectionTypeResolver $mqConnectionTypeResolver = null, + LoggerInterface $logger = null + ) { +@@ -93,7 +88,7 @@ class ConsumersRunner + $this->consumerConfig = $consumerConfig; + $this->deploymentConfig = $deploymentConfig; + $this->shellBackground = $shellBackground; +- $this->pidConsumerManager = $pidConsumerManager; ++ $this->lockManager = $lockManager; + $this->mqConnectionTypeResolver = $mqConnectionTypeResolver + ?: ObjectManager::getInstance()->get(ConnectionTypeResolver::class); + $this->logger = $logger +@@ -120,11 +115,9 @@ class ConsumersRunner + continue; + } + +- $consumerName = $consumer->getName(); +- + $arguments = [ +- $consumerName, +- '--pid-file-path=' . $this->getPidFilePath($consumerName), ++ $consumer->getName(), ++ '--single-thread' + ]; + + if ($maxMessages) { +@@ -154,7 +147,7 @@ class ConsumersRunner + return false; + } + +- if ($this->pidConsumerManager->isRun($this->getPidFilePath($consumerName))) { ++ if ($this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore + return false; + } + +@@ -162,28 +155,17 @@ class ConsumersRunner + try { + $this->mqConnectionTypeResolver->getConnectionType($connectionName); + } catch (\LogicException $e) { +- $this->logger->info(sprintf( +- 'Consumer "%s" skipped as required connection "%s" is not configured. %s', +- $consumerName, +- $connectionName, +- $e->getMessage() +- )); ++ $this->logger->info( ++ sprintf( ++ 'Consumer "%s" skipped as required connection "%s" is not configured. %s', ++ $consumerName, ++ $connectionName, ++ $e->getMessage() ++ ) ++ ); + return false; + } + + return true; + } +- +- /** +- * Returns default path to file with PID by consumers name +- * +- * @param string $consumerName The consumers name +- * @return string The path to file with PID +- */ +- private function getPidFilePath($consumerName) +- { +- $sanitizedHostname = preg_replace('/[^a-z0-9]/i', '', gethostname()); +- +- return $consumerName . '-' . $sanitizedHostname . static::PID_FILE_EXT; +- } + } +diff --git a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php +deleted file mode 100644 +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php ++++ /dev/null +@@ -1,127 +0,0 @@ +-filesystem = $filesystem; +- } +- +- /** +- * Checks if consumer process is run by pid from pidFile +- * +- * @param string $pidFilePath The path to file with PID +- * @return bool Returns true if consumer process is run +- * @throws FileSystemException +- */ +- public function isRun($pidFilePath) +- { +- $pid = $this->getPid($pidFilePath); +- if ($pid) { +- if (function_exists('posix_getpgid')) { +- return (bool) posix_getpgid($pid); +- } else { +- return $this->checkIsProcessExists($pid); +- } +- } +- +- return false; +- } +- +- /** +- * Checks that process is running +- * +- * If php function exec is not available throws RuntimeException +- * If shell command returns non-zero code and this code is not 1 throws RuntimeException +- * +- * @param int $pid A pid of process +- * @return bool Returns true if consumer process is run +- * @throws \RuntimeException +- * @SuppressWarnings(PHPMD.UnusedLocalVariable) +- */ +- private function checkIsProcessExists($pid) +- { +- if (!function_exists('exec')) { +- throw new \RuntimeException('Function exec is not available'); +- } +- +- exec(escapeshellcmd('ps -p ' . $pid), $output, $code); +- +- $code = (int) $code; +- +- switch ($code) { +- case 0: +- return true; +- break; +- case 1: +- return false; +- break; +- default: +- throw new \RuntimeException('Exec returned non-zero code', $code); +- break; +- } +- } +- +- /** +- * Returns pid by pidFile path +- * +- * @param string $pidFilePath The path to file with PID +- * @return int Returns pid if pid file exists for consumer else returns 0 +- * @throws FileSystemException +- */ +- public function getPid($pidFilePath) +- { +- /** @var ReadInterface $directory */ +- $directory = $this->filesystem->getDirectoryRead(DirectoryList::VAR_DIR); +- +- if ($directory->isExist($pidFilePath)) { +- return (int) $directory->readFile($pidFilePath); +- } +- +- return 0; +- } +- +- /** +- * Saves pid of current process to file +- * +- * @param string $pidFilePath The path to file with pid +- * @throws FileSystemException +- */ +- public function savePid($pidFilePath) +- { +- /** @var WriteInterface $directory */ +- $directory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); +- $directory->writeFile($pidFilePath, function_exists('posix_getpid') ? posix_getpid() : getmypid(), 'w'); +- } +-} diff --git a/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.1.patch b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.1.patch new file mode 100644 index 0000000..8165953 --- /dev/null +++ b/patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.1.patch @@ -0,0 +1,447 @@ +diff -Nuar a/vendor/magento/framework/Lock/Backend/Database.php b/vendor/magento/framework/Lock/Backend/Database.php +--- a/vendor/magento/framework/Lock/Backend/Database.php ++++ b/vendor/magento/framework/Lock/Backend/Database.php +@@ -19,6 +19,14 @@ use Magento\Framework\Phrase; + */ + class Database implements \Magento\Framework\Lock\LockManagerInterface + { ++ /** ++ * Max time for lock is 1 week ++ * ++ * MariaDB does not support negative timeout value to get infinite timeout, ++ * so we set 1 week for lock timeout ++ */ ++ private const MAX_LOCK_TIME = 604800; ++ + /** + * @var ResourceConnection + */ +@@ -87,7 +95,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + + $result = (bool)$this->resource->getConnection()->query( + "SELECT GET_LOCK(?, ?);", +- [(string)$name, (int)$timeout] ++ [$name, $timeout < 0 ? self::MAX_LOCK_TIME : $timeout] + )->fetchColumn(); + + if ($result === true) { +@@ -143,7 +151,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface + + return (bool)$this->resource->getConnection()->query( + "SELECT IS_USED_LOCK(?);", +- [(string)$name] ++ [$name] + )->fetchColumn(); + } + +diff -Nuar a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +--- a/vendor/magento/module-message-queue/Console/StartConsumerCommand.php ++++ b/vendor/magento/module-message-queue/Console/StartConsumerCommand.php +@@ -11,7 +11,7 @@ use Symfony\Component\Console\Input\InputInterface; + use Symfony\Component\Console\Input\InputOption; + use Symfony\Component\Console\Output\OutputInterface; + use Magento\Framework\MessageQueue\ConsumerFactory; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Command for starting MessageQueue consumers. +@@ -22,6 +22,7 @@ class StartConsumerCommand extends Command + const OPTION_NUMBER_OF_MESSAGES = 'max-messages'; + const OPTION_BATCH_SIZE = 'batch-size'; + const OPTION_AREACODE = 'area-code'; ++ const OPTION_SINGLE_THREAD = 'single-thread'; + const PID_FILE_PATH = 'pid-file-path'; + const COMMAND_QUEUE_CONSUMERS_START = 'queue:consumers:start'; + +@@ -36,9 +37,9 @@ class StartConsumerCommand extends Command + private $appState; + + /** +- * @var PidConsumerManager ++ * @var LockManagerInterface + */ +- private $pidConsumerManager; ++ private $lockManager; + + /** + * StartConsumerCommand constructor. +@@ -47,23 +48,23 @@ class StartConsumerCommand extends Command + * @param \Magento\Framework\App\State $appState + * @param ConsumerFactory $consumerFactory + * @param string $name +- * @param PidConsumerManager $pidConsumerManager ++ * @param LockManagerInterface $lockManager + */ + public function __construct( + \Magento\Framework\App\State $appState, + ConsumerFactory $consumerFactory, + $name = null, +- PidConsumerManager $pidConsumerManager = null ++ LockManagerInterface $lockManager = null + ) { + $this->appState = $appState; + $this->consumerFactory = $consumerFactory; +- $this->pidConsumerManager = $pidConsumerManager ?: \Magento\Framework\App\ObjectManager::getInstance() +- ->get(PidConsumerManager::class); ++ $this->lockManager = $lockManager ?: \Magento\Framework\App\ObjectManager::getInstance() ++ ->get(LockManagerInterface::class); + parent::__construct($name); + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function execute(InputInterface $input, OutputInterface $output) + { +@@ -71,30 +72,36 @@ class StartConsumerCommand extends Command + $numberOfMessages = $input->getOption(self::OPTION_NUMBER_OF_MESSAGES); + $batchSize = (int)$input->getOption(self::OPTION_BATCH_SIZE); + $areaCode = $input->getOption(self::OPTION_AREACODE); +- $pidFilePath = $input->getOption(self::PID_FILE_PATH); + +- if ($pidFilePath && $this->pidConsumerManager->isRun($pidFilePath)) { +- $output->writeln('Consumer with the same PID is running'); +- return \Magento\Framework\Console\Cli::RETURN_FAILURE; ++ if ($input->getOption(self::PID_FILE_PATH)) { ++ $input->setOption(self::OPTION_SINGLE_THREAD, true); + } + +- if ($pidFilePath) { +- $this->pidConsumerManager->savePid($pidFilePath); ++ $singleThread = $input->getOption(self::OPTION_SINGLE_THREAD); ++ ++ if ($singleThread && $this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore ++ $output->writeln('Consumer with the same name is running'); ++ return \Magento\Framework\Console\Cli::RETURN_FAILURE; + } + +- if ($areaCode !== null) { +- $this->appState->setAreaCode($areaCode); +- } else { +- $this->appState->setAreaCode('global'); ++ if ($singleThread) { ++ $this->lockManager->lock(md5($consumerName)); //phpcs:ignore + } + ++ $this->appState->setAreaCode($areaCode ?? 'global'); ++ + $consumer = $this->consumerFactory->get($consumerName, $batchSize); + $consumer->process($numberOfMessages); ++ ++ if ($singleThread) { ++ $this->lockManager->unlock(md5($consumerName)); //phpcs:ignore ++ } ++ + return \Magento\Framework\Console\Cli::RETURN_SUCCESS; + } + + /** +- * {@inheritdoc} ++ * @inheritdoc + */ + protected function configure() + { +@@ -125,11 +132,17 @@ class StartConsumerCommand extends Command + 'The preferred area (global, adminhtml, etc...) ' + . 'default is global.' + ); ++ $this->addOption( ++ self::OPTION_SINGLE_THREAD, ++ null, ++ InputOption::VALUE_NONE, ++ 'This option prevents running multiple copies of one consumer simultaneously.' ++ ); + $this->addOption( + self::PID_FILE_PATH, + null, + InputOption::VALUE_REQUIRED, +- 'The file path for saving PID' ++ 'The file path for saving PID (This option is deprecated, use --single-thread instead)' + ); + $this->setHelp( + <<%command.full_name% someConsumer --area-code='adminhtml' ++ ++To do not run multiple copies of one consumer simultaneously: ++ ++ %command.full_name% someConsumer --single-thread' + +-To save PID enter path: ++To save PID enter path (This option is deprecated, use --single-thread instead): + + %command.full_name% someConsumer --pid-file-path='/var/someConsumer.pid' + HELP +diff -Nuar a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php ++++ b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner.php +@@ -13,18 +13,13 @@ use Magento\Framework\MessageQueue\Consumer\ConfigInterface as ConsumerConfigInt + use Magento\Framework\App\DeploymentConfig; + use Psr\Log\LoggerInterface; + use Symfony\Component\Process\PhpExecutableFinder; +-use Magento\MessageQueue\Model\Cron\ConsumersRunner\PidConsumerManager; ++use Magento\Framework\Lock\LockManagerInterface; + + /** + * Class for running consumers processes by cron + */ + class ConsumersRunner + { +- /** +- * Extension of PID file +- */ +- const PID_FILE_EXT = '.pid'; +- + /** + * Shell command line wrapper for executing command in background + * +@@ -53,13 +48,6 @@ class ConsumersRunner + */ + private $phpExecutableFinder; + +- /** +- * The class for checking status of process by PID +- * +- * @var PidConsumerManager +- */ +- private $pidConsumerManager; +- + /** + * @var ConnectionTypeResolver + */ +@@ -70,13 +58,20 @@ class ConsumersRunner + */ + private $logger; + ++ /** ++ * Lock Manager ++ * ++ * @var LockManagerInterface ++ */ ++ private $lockManager; ++ + /** + * @param PhpExecutableFinder $phpExecutableFinder The executable finder specifically designed + * for the PHP executable + * @param ConsumerConfigInterface $consumerConfig The consumer config provider + * @param DeploymentConfig $deploymentConfig The application deployment configuration + * @param ShellInterface $shellBackground The shell command line wrapper for executing command in background +- * @param PidConsumerManager $pidConsumerManager The class for checking status of process by PID ++ * @param LockManagerInterface $lockManager The lock manager + * @param ConnectionTypeResolver $mqConnectionTypeResolver Consumer connection resolver + * @param LoggerInterface $logger Logger + */ +@@ -85,7 +80,7 @@ class ConsumersRunner + ConsumerConfigInterface $consumerConfig, + DeploymentConfig $deploymentConfig, + ShellInterface $shellBackground, +- PidConsumerManager $pidConsumerManager, ++ LockManagerInterface $lockManager, + ConnectionTypeResolver $mqConnectionTypeResolver = null, + LoggerInterface $logger = null + ) { +@@ -93,7 +88,7 @@ class ConsumersRunner + $this->consumerConfig = $consumerConfig; + $this->deploymentConfig = $deploymentConfig; + $this->shellBackground = $shellBackground; +- $this->pidConsumerManager = $pidConsumerManager; ++ $this->lockManager = $lockManager; + $this->mqConnectionTypeResolver = $mqConnectionTypeResolver + ?: ObjectManager::getInstance()->get(ConnectionTypeResolver::class); + $this->logger = $logger +@@ -120,11 +115,9 @@ class ConsumersRunner + continue; + } + +- $consumerName = $consumer->getName(); +- + $arguments = [ +- $consumerName, +- '--pid-file-path=' . $this->getPidFilePath($consumerName), ++ $consumer->getName(), ++ '--single-thread' + ]; + + if ($maxMessages) { +@@ -154,7 +147,7 @@ class ConsumersRunner + return false; + } + +- if ($this->pidConsumerManager->isRun($this->getPidFilePath($consumerName))) { ++ if ($this->lockManager->isLocked(md5($consumerName))) { //phpcs:ignore + return false; + } + +@@ -162,28 +155,17 @@ class ConsumersRunner + try { + $this->mqConnectionTypeResolver->getConnectionType($connectionName); + } catch (\LogicException $e) { +- $this->logger->info(sprintf( +- 'Consumer "%s" skipped as required connection "%s" is not configured. %s', +- $consumerName, +- $connectionName, +- $e->getMessage() +- )); ++ $this->logger->info( ++ sprintf( ++ 'Consumer "%s" skipped as required connection "%s" is not configured. %s', ++ $consumerName, ++ $connectionName, ++ $e->getMessage() ++ ) ++ ); + return false; + } + + return true; + } +- +- /** +- * Returns default path to file with PID by consumers name +- * +- * @param string $consumerName The consumers name +- * @return string The path to file with PID +- */ +- private function getPidFilePath($consumerName) +- { +- $sanitizedHostname = preg_replace('/[^a-z0-9]/i', '', gethostname()); +- +- return $consumerName . '-' . $sanitizedHostname . static::PID_FILE_EXT; +- } + } +diff --git a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php b/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php +deleted file mode 100644 +--- a/vendor/magento/module-message-queue/Model/Cron/ConsumersRunner/PidConsumerManager.php ++++ /dev/null +@@ -1,127 +0,0 @@ +-filesystem = $filesystem; +- } +- +- /** +- * Checks if consumer process is run by pid from pidFile +- * +- * @param string $pidFilePath The path to file with PID +- * @return bool Returns true if consumer process is run +- * @throws FileSystemException +- */ +- public function isRun($pidFilePath) +- { +- $pid = $this->getPid($pidFilePath); +- if ($pid) { +- if (function_exists('posix_getpgid')) { +- return (bool) posix_getpgid($pid); +- } else { +- return $this->checkIsProcessExists($pid); +- } +- } +- +- return false; +- } +- +- /** +- * Checks that process is running +- * +- * If php function exec is not available throws RuntimeException +- * If shell command returns non-zero code and this code is not 1 throws RuntimeException +- * +- * @param int $pid A pid of process +- * @return bool Returns true if consumer process is run +- * @throws \RuntimeException +- * @SuppressWarnings(PHPMD.UnusedLocalVariable) +- */ +- private function checkIsProcessExists($pid) +- { +- if (!function_exists('exec')) { +- throw new \RuntimeException('Function exec is not available'); +- } +- +- exec(escapeshellcmd('ps -p ' . $pid), $output, $code); +- +- $code = (int) $code; +- +- switch ($code) { +- case 0: +- return true; +- break; +- case 1: +- return false; +- break; +- default: +- throw new \RuntimeException('Exec returned non-zero code', $code); +- break; +- } +- } +- +- /** +- * Returns pid by pidFile path +- * +- * @param string $pidFilePath The path to file with PID +- * @return int Returns pid if pid file exists for consumer else returns 0 +- * @throws FileSystemException +- */ +- public function getPid($pidFilePath) +- { +- /** @var ReadInterface $directory */ +- $directory = $this->filesystem->getDirectoryRead(DirectoryList::VAR_DIR); +- +- if ($directory->isExist($pidFilePath)) { +- return (int) $directory->readFile($pidFilePath); +- } +- +- return 0; +- } +- +- /** +- * Saves pid of current process to file +- * +- * @param string $pidFilePath The path to file with pid +- * @throws FileSystemException +- */ +- public function savePid($pidFilePath) +- { +- /** @var WriteInterface $directory */ +- $directory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); +- $directory->writeFile($pidFilePath, function_exists('posix_getpid') ? posix_getpid() : getmypid(), 'w'); +- } +-} diff --git a/patches/MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.2.0.patch b/patches/MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.2.0.patch new file mode 100644 index 0000000..2a019dd --- /dev/null +++ b/patches/MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.2.0.patch @@ -0,0 +1,170 @@ +diff -Naur a/vendor/magento/framework-message-queue/CallbackInvoker.php b/vendor/magento/framework-message-queue/CallbackInvoker.php +--- a/vendor/magento/framework-message-queue/CallbackInvoker.php ++++ b/vendor/magento/framework-message-queue/CallbackInvoker.php +@@ -6,11 +6,28 @@ + + namespace Magento\Framework\MessageQueue; + ++use Magento\Framework\App\DeploymentConfig; ++ + /** + * Class CallbackInvoker to invoke callbacks for consumer classes + */ + class CallbackInvoker + { ++ /** ++ * @var DeploymentConfig ++ */ ++ private $deploymentConfig; ++ ++ /** ++ * CallbackInvoker constructor. ++ * @param DeploymentConfig $deploymentConfig ++ */ ++ public function __construct( ++ DeploymentConfig $deploymentConfig ++ ) { ++ $this->deploymentConfig = $deploymentConfig; ++ } ++ + /** + * Run short running process + * +@@ -24,8 +41,23 @@ class CallbackInvoker + for ($i = $maxNumberOfMessages; $i > 0; $i--) { + do { + $message = $queue->dequeue(); +- } while ($message === null && (sleep(1) === 0)); ++ } while ($message === null && $this->isWaitingNextMessage() && (sleep(1) === 0)); ++ ++ if ($message === null) { ++ break; ++ } ++ + $callback($message); + } + } ++ ++ /** ++ * Checks if consumers should wait for message from the queue ++ * ++ * @return bool ++ */ ++ private function isWaitingNextMessage(): bool ++ { ++ return $this->deploymentConfig->get('queue/consumers_wait_for_messages', 1) === 1; ++ } + } +diff -Naur a/vendor/magento/module-message-queue/Setup/ConfigOptionsList.php b/vendor/magento/module-message-queue/Setup/ConfigOptionsList.php +new file mode 100644 +--- /dev/null ++++ b/vendor/magento/module-message-queue/Setup/ConfigOptionsList.php +@@ -0,0 +1,108 @@ ++selectOptions, ++ self::CONFIG_PATH_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES, ++ 'Should consumers wait for a message from the queue? 1 - Yes, 0 - No', ++ self::DEFAULT_CONSUMERS_WAIT_FOR_MESSAGES ++ ), ++ ]; ++ } ++ ++ /** ++ * @inheritdoc ++ * @SuppressWarnings(PHPMD.UnusedFormalParameter) ++ */ ++ public function createConfig(array $data, DeploymentConfig $deploymentConfig) ++ { ++ $configData = new ConfigData(ConfigFilePool::APP_ENV); ++ ++ if (!$this->isDataEmpty($data, self::INPUT_KEY_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES)) { ++ $configData->set( ++ self::CONFIG_PATH_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES, ++ (int)$data[self::INPUT_KEY_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES] ++ ); ++ } ++ ++ return [$configData]; ++ } ++ ++ /** ++ * @inheritdoc ++ */ ++ public function validate(array $options, DeploymentConfig $deploymentConfig) ++ { ++ $errors = []; ++ ++ if (!$this->isDataEmpty($options, self::INPUT_KEY_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES) ++ && !in_array($options[self::INPUT_KEY_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES], $this->selectOptions)) { ++ $errors[] = 'You can use only 1 or 0 for ' . self::INPUT_KEY_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES . ' option'; ++ } ++ ++ return $errors; ++ } ++ ++ /** ++ * Check if data ($data) with key ($key) is empty ++ * ++ * @param array $data ++ * @param string $key ++ * @return bool ++ */ ++ private function isDataEmpty(array $data, $key) ++ { ++ if (isset($data[$key]) && $data[$key] !== '') { ++ return false; ++ } ++ ++ return true; ++ } ++} diff --git a/patches/MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.3.2.patch b/patches/MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.3.2.patch new file mode 100644 index 0000000..eb1b785 --- /dev/null +++ b/patches/MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.3.2.patch @@ -0,0 +1,181 @@ +diff -Naur a/vendor/magento/framework-message-queue/CallbackInvoker.php b/vendor/magento/framework-message-queue/CallbackInvoker.php +--- a/vendor/magento/framework-message-queue/CallbackInvoker.php ++++ b/vendor/magento/framework-message-queue/CallbackInvoker.php +@@ -8,6 +8,7 @@ namespace Magento\Framework\MessageQueue; + + use Magento\Framework\MessageQueue\PoisonPill\PoisonPillCompareInterface; + use Magento\Framework\MessageQueue\PoisonPill\PoisonPillReadInterface; ++use Magento\Framework\App\DeploymentConfig; + + /** + * Class CallbackInvoker to invoke callbacks for consumer classes +@@ -29,16 +30,24 @@ class CallbackInvoker implements CallbackInvokerInterface + */ + private $poisonPillCompare; + ++ /** ++ * @var DeploymentConfig ++ */ ++ private $deploymentConfig; ++ + /** + * @param PoisonPillReadInterface $poisonPillRead + * @param PoisonPillCompareInterface $poisonPillCompare ++ * @param DeploymentConfig $deploymentConfig + */ + public function __construct( + PoisonPillReadInterface $poisonPillRead, +- PoisonPillCompareInterface $poisonPillCompare ++ PoisonPillCompareInterface $poisonPillCompare, ++ DeploymentConfig $deploymentConfig + ) { + $this->poisonPillRead = $poisonPillRead; + $this->poisonPillCompare = $poisonPillCompare; ++ $this->deploymentConfig = $deploymentConfig; + } + + /** +@@ -56,13 +65,29 @@ class CallbackInvoker implements CallbackInvokerInterface + do { + $message = $queue->dequeue(); + // phpcs:ignore Magento2.Functions.DiscouragedFunction +- } while ($message === null && (sleep(1) === 0)); ++ } while ($message === null && $this->isWaitingNextMessage() && (sleep(1) === 0)); ++ ++ if ($message === null) { ++ break; ++ } ++ + if (false === $this->poisonPillCompare->isLatestVersion($this->poisonPillVersion)) { + $queue->reject($message); + // phpcs:ignore Magento2.Security.LanguageConstruct.ExitUsage + exit(0); + } ++ + $callback($message); + } + } ++ ++ /** ++ * Checks if consumers should wait for message from the queue ++ * ++ * @return bool ++ */ ++ private function isWaitingNextMessage(): bool ++ { ++ return $this->deploymentConfig->get('queue/consumers_wait_for_messages', 1) === 1; ++ } + } +diff -Naur a/vendor/magento/module-message-queue/Setup/ConfigOptionsList.php b/vendor/magento/module-message-queue/Setup/ConfigOptionsList.php +new file mode 100644 +--- /dev/null ++++ b/vendor/magento/module-message-queue/Setup/ConfigOptionsList.php +@@ -0,0 +1,108 @@ ++selectOptions, ++ self::CONFIG_PATH_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES, ++ 'Should consumers wait for a message from the queue? 1 - Yes, 0 - No', ++ self::DEFAULT_CONSUMERS_WAIT_FOR_MESSAGES ++ ), ++ ]; ++ } ++ ++ /** ++ * @inheritdoc ++ * @SuppressWarnings(PHPMD.UnusedFormalParameter) ++ */ ++ public function createConfig(array $data, DeploymentConfig $deploymentConfig) ++ { ++ $configData = new ConfigData(ConfigFilePool::APP_ENV); ++ ++ if (!$this->isDataEmpty($data, self::INPUT_KEY_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES)) { ++ $configData->set( ++ self::CONFIG_PATH_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES, ++ (int)$data[self::INPUT_KEY_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES] ++ ); ++ } ++ ++ return [$configData]; ++ } ++ ++ /** ++ * @inheritdoc ++ */ ++ public function validate(array $options, DeploymentConfig $deploymentConfig) ++ { ++ $errors = []; ++ ++ if (!$this->isDataEmpty($options, self::INPUT_KEY_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES) ++ && !in_array($options[self::INPUT_KEY_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES], $this->selectOptions)) { ++ $errors[] = 'You can use only 1 or 0 for ' . self::INPUT_KEY_QUEUE_CONSUMERS_WAIT_FOR_MESSAGES . ' option'; ++ } ++ ++ return $errors; ++ } ++ ++ /** ++ * Check if data ($data) with key ($key) is empty ++ * ++ * @param array $data ++ * @param string $key ++ * @return bool ++ */ ++ private function isDataEmpty(array $data, $key) ++ { ++ if (isset($data[$key]) && $data[$key] !== '') { ++ return false; ++ } ++ ++ return true; ++ } ++} diff --git a/patches/MAGECLOUD-414__remove_unnecessary_permission_checks__2.1.4.patch b/patches/MAGECLOUD-414__remove_unnecessary_permission_checks__2.1.4.patch new file mode 100644 index 0000000..7592814 --- /dev/null +++ b/patches/MAGECLOUD-414__remove_unnecessary_permission_checks__2.1.4.patch @@ -0,0 +1,43 @@ +diff -Naur b/vendor/magento/framework/Setup/FilePermissions.php a/vendor/magento/framework/Setup/FilePermissions.php +--- b/vendor/magento/framework/Setup/FilePermissions.php 2016-09-23 16:01:12.000000000 -0500 ++++ a/vendor/magento/framework/Setup/FilePermissions.php 2016-09-23 16:22:09.000000000 -0500 +@@ -233,26 +233,8 @@ + */ + public function getMissingWritablePathsForInstallation($associative = false) + { +- $required = $this->getInstallationWritableDirectories(); +- $current = $this->getInstallationCurrentWritableDirectories(); +- $missingPaths = []; +- foreach (array_diff($required, $current) as $missingPath) { +- if (isset($this->nonWritablePathsInDirectories[$missingPath])) { +- if ($associative) { +- $missingPaths[$missingPath] = $this->nonWritablePathsInDirectories[$missingPath]; +- } else { +- $missingPaths = array_merge( +- $missingPaths, +- $this->nonWritablePathsInDirectories[$missingPath] +- ); +- } +- } +- } +- if ($associative) { +- $required = array_flip($required); +- $missingPaths = array_merge($required, $missingPaths); +- } +- return $missingPaths; ++ // Unnecessary check in controlled environment ++ return []; + } + + /** +@@ -275,8 +257,7 @@ + */ + public function getUnnecessaryWritableDirectoriesForApplication() + { +- $required = $this->getApplicationNonWritableDirectories(); +- $current = $this->getApplicationCurrentNonWritableDirectories(); +- return array_diff($required, $current); ++ // Unnecessary check in controlled environment ++ return []; + } + } diff --git a/patches/MAGECLOUD-414__remove_unnecessary_permission_checks__2.2.0.patch b/patches/MAGECLOUD-414__remove_unnecessary_permission_checks__2.2.0.patch new file mode 100644 index 0000000..7592814 --- /dev/null +++ b/patches/MAGECLOUD-414__remove_unnecessary_permission_checks__2.2.0.patch @@ -0,0 +1,43 @@ +diff -Naur b/vendor/magento/framework/Setup/FilePermissions.php a/vendor/magento/framework/Setup/FilePermissions.php +--- b/vendor/magento/framework/Setup/FilePermissions.php 2016-09-23 16:01:12.000000000 -0500 ++++ a/vendor/magento/framework/Setup/FilePermissions.php 2016-09-23 16:22:09.000000000 -0500 +@@ -233,26 +233,8 @@ + */ + public function getMissingWritablePathsForInstallation($associative = false) + { +- $required = $this->getInstallationWritableDirectories(); +- $current = $this->getInstallationCurrentWritableDirectories(); +- $missingPaths = []; +- foreach (array_diff($required, $current) as $missingPath) { +- if (isset($this->nonWritablePathsInDirectories[$missingPath])) { +- if ($associative) { +- $missingPaths[$missingPath] = $this->nonWritablePathsInDirectories[$missingPath]; +- } else { +- $missingPaths = array_merge( +- $missingPaths, +- $this->nonWritablePathsInDirectories[$missingPath] +- ); +- } +- } +- } +- if ($associative) { +- $required = array_flip($required); +- $missingPaths = array_merge($required, $missingPaths); +- } +- return $missingPaths; ++ // Unnecessary check in controlled environment ++ return []; + } + + /** +@@ -275,8 +257,7 @@ + */ + public function getUnnecessaryWritableDirectoriesForApplication() + { +- $required = $this->getApplicationNonWritableDirectories(); +- $current = $this->getApplicationCurrentNonWritableDirectories(); +- return array_diff($required, $current); ++ // Unnecessary check in controlled environment ++ return []; + } + } diff --git a/patches/MAGECLOUD-589__load_appropriate_js_files__2.1.4.patch b/patches/MAGECLOUD-589__load_appropriate_js_files__2.1.4.patch new file mode 100644 index 0000000..b9f1718 --- /dev/null +++ b/patches/MAGECLOUD-589__load_appropriate_js_files__2.1.4.patch @@ -0,0 +1,26 @@ +MAGECLOUD-589 MAGETWO-64955 +diff -Naur a/vendor/magento/framework/View/Asset/File/FallbackContext.php b/vendor/magento/framework/View/Asset/File/FallbackContext.php +--- a/vendor/magento/framework/View/Asset/File/FallbackContext.php 2016-10-15 12:38:21.595266000 +0000 ++++ b/vendor/magento/framework/View/Asset/File/FallbackContext.php 2016-10-15 12:39:13.587266000 +0000 +@@ -103,6 +103,6 @@ + */ + public function getConfigPath() + { +- return $this->getPath() . ($this->isSecure ? '/' . self::SECURE_PATH : ''); ++ return $this->getPath(); + } + } + +diff -Nuar a/vendor/magento/framework/View/Asset/Repository.php b/vendor/magento/framework/View/Asset/Repository.php +--- a/vendor/magento/framework/View/Asset/Repository.php 2017-02-16 16:50:18.000000000 +0000 ++++ b/vendor/magento/framework/View/Asset/Repository.php 2017-02-22 14:29:40.000000000 +0000 +@@ -269,8 +269,7 @@ + 'baseUrl' => $url, + 'areaType' => $area, + 'themePath' => $themePath, +- 'localeCode' => $locale, +- 'isSecure' => $isSecure ++ 'localeCode' => $locale + ] + ); + } diff --git a/patches/MAGETWO-45357__avoid_nonexistent_setup_area__2.1.4.patch b/patches/MAGETWO-45357__avoid_nonexistent_setup_area__2.1.4.patch new file mode 100644 index 0000000..443388f --- /dev/null +++ b/patches/MAGETWO-45357__avoid_nonexistent_setup_area__2.1.4.patch @@ -0,0 +1,15 @@ +Ticket MAGETWO-45357 +diff -Nuar a/vendor/magento/framework/App/ObjectManager/ConfigLoader/Compiled.php b/vendor/magento/framework/App/ObjectManager/ConfigLoader/Compiled.php +index 844d3f0..d087d07 100644 +--- a/vendor/magento/framework/App/ObjectManager/ConfigLoader/Compiled.php ++++ b/vendor/magento/framework/App/ObjectManager/ConfigLoader/Compiled.php +@@ -37,6 +37,9 @@ class Compiled implements ConfigLoaderInterface + */ + public static function getFilePath($area) + { ++ if ($area == 'setup') { ++ $area = 'global'; ++ } + return BP . '/var/di/' . $area . '.ser'; + } + } diff --git a/patches/MAGETWO-53941__fix_enterprise_payment_codes__2.1.8.patch b/patches/MAGETWO-53941__fix_enterprise_payment_codes__2.1.8.patch new file mode 100644 index 0000000..1f75f0d --- /dev/null +++ b/patches/MAGETWO-53941__fix_enterprise_payment_codes__2.1.8.patch @@ -0,0 +1,93 @@ +<+>UTF-8 +=================================================================== +diff -Nuar a/vendor/magento/module-braintree/Gateway/Request/ChannelDataBuilder.php b/vendor/magento/module-braintree/Gateway/Request/ChannelDataBuilder.php +--- a/vendor/magento/module-braintree/Gateway/Request/ChannelDataBuilder.php ++++ b/vendor/magento/module-braintree/Gateway/Request/ChannelDataBuilder.php +@@ -26,7 +26,7 @@ + /** + * @var string + */ +- private static $channelValue = 'Magento2_Cart_%s_BT'; ++ private static $channelValue = 'Magento_Enterprise_Cloud_BT'; + + /** + * Constructor +@@ -44,7 +44,7 @@ + public function build(array $buildSubject) + { + return [ +- self::$channel => sprintf(self::$channelValue, $this->productMetadata->getEdition()) ++ self::$channel => self::$channelValue + ]; + } + } +<+>UTF-8 +=================================================================== +diff -Nuar a/vendor/magento/module-braintree/Test/Unit/Gateway/Request/ChannelDataBuilderTest.php b/vendor/magento/module-braintree/Test/Unit/Gateway/Request/ChannelDataBuilderTest.php +--- a/vendor/magento/module-braintree/Test/Unit/Gateway/Request/ChannelDataBuilderTest.php ++++ b/vendor/magento/module-braintree/Test/Unit/Gateway/Request/ChannelDataBuilderTest.php +@@ -40,7 +40,7 @@ + public function testBuild($edition, array $expected) + { + $buildSubject = []; +- $this->productMetadataMock->expects(static::once()) ++ $this->productMetadataMock->expects(static::never()) + ->method('getEdition') + ->willReturn($edition); + +@@ -54,8 +54,8 @@ + public function buildDataProvider() + { + return [ +- ['FirstEdition', ['channel' => 'Magento2_Cart_FirstEdition_BT']], +- ['SecondEdition', ['channel' => 'Magento2_Cart_SecondEdition_BT']], ++ ['FirstEdition', ['channel' => 'Magento_Enterprise_Cloud_BT']], ++ ['SecondEdition', ['channel' => 'Magento_Enterprise_Cloud_BT']], + ]; + } + } +<+>UTF-8 +=================================================================== +diff -Nuar a/vendor/magento/module-paypal/Model/AbstractConfig.php b/vendor/magento/module-paypal/Model/AbstractConfig.php +--- a/vendor/magento/module-paypal/Model/AbstractConfig.php ++++ b/vendor/magento/module-paypal/Model/AbstractConfig.php +@@ -59,7 +59,7 @@ + /** + * @var string + */ +- private static $bnCode = 'Magento_Cart_%s'; ++ private static $bnCode = 'Magento_Enterprise_Cloud'; + + /** + * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig +@@ -335,7 +335,7 @@ + */ + public function getBuildNotationCode() + { +- return sprintf(self::$bnCode, $this->getProductMetadata()->getEdition()); ++ return self::$bnCode; + } + + /** +<+>UTF-8 +=================================================================== +diff -Nuar a/vendor/magento/module-paypal/Test/Unit/Model/AbstractConfigTest.php b/vendor/magento/module-paypal/Test/Unit/Model/AbstractConfigTest.php +--- a/vendor/magento/module-paypal/Test/Unit/Model/AbstractConfigTest.php ++++ b/vendor/magento/module-paypal/Test/Unit/Model/AbstractConfigTest.php +@@ -293,7 +293,7 @@ + public function testGetBuildNotationCode() + { + $productMetadata = $this->getMock(ProductMetadataInterface::class, [], [], '', false); +- $productMetadata->expects($this->once()) ++ $productMetadata->expects($this->never()) + ->method('getEdition') + ->will($this->returnValue('SomeEdition')); + +@@ -304,6 +304,6 @@ + $productMetadata + ); + +- $this->assertEquals('Magento_Cart_SomeEdition', $this->config->getBuildNotationCode()); ++ $this->assertEquals('Magento_Enterprise_Cloud', $this->config->getBuildNotationCode()); + } + } diff --git a/patches/MAGETWO-56675__dont_skip_setup_scoped_plugins__2.1.4.patch b/patches/MAGETWO-56675__dont_skip_setup_scoped_plugins__2.1.4.patch new file mode 100644 index 0000000..b1bf5a4 --- /dev/null +++ b/patches/MAGETWO-56675__dont_skip_setup_scoped_plugins__2.1.4.patch @@ -0,0 +1,20 @@ +Patch regarding MAGETWO-56675. +diff -Nuar a/vendor/magento/framework/Interception/PluginList/PluginList.php b/vendor/magento/framework/Interception/PluginList/PluginList.php +--- a/vendor/magento/framework/Interception/PluginList/PluginList.php ++++ b/vendor/magento/framework/Interception/PluginList/PluginList.php +@@ -277,6 +277,7 @@ class PluginList extends Scoped implements InterceptionPluginList + $virtualTypes = []; + foreach ($this->_scopePriorityScheme as $scopeCode) { + if (false == isset($this->_loadedScopes[$scopeCode])) { ++ $this->_loadedScopes[$scopeCode] = true; + $data = $this->_reader->read($scopeCode); + unset($data['preferences']); + if (!count($data)) { +@@ -285,7 +286,6 @@ class PluginList extends Scoped implements InterceptionPluginList + $this->_inherited = []; + $this->_processed = []; + $this->merge($data); +- $this->_loadedScopes[$scopeCode] = true; + foreach ($data as $class => $config) { + if (isset($config['type'])) { + $virtualTypes[] = $class; diff --git a/patches/MAGETWO-57413__move_vendor_path_autoloader__2.1.4.patch b/patches/MAGETWO-57413__move_vendor_path_autoloader__2.1.4.patch new file mode 100644 index 0000000..c30379b --- /dev/null +++ b/patches/MAGETWO-57413__move_vendor_path_autoloader__2.1.4.patch @@ -0,0 +1,37 @@ +Ticket MAGETWO-57413 +diff -Naur a/app/autoload.php b/app/autoload.php +index b817baf..1d1873d 100644 +--- a/app/autoload.php ++++ b/app/autoload.php +@@ -13,16 +13,7 @@ use Magento\Framework\Autoload\ClassLoaderWrapper; + */ + define('BP', dirname(__DIR__)); + +-define('VENDOR_PATH', BP . '/app/etc/vendor_path.php'); +- +-if (!file_exists(VENDOR_PATH)) { +- throw new \Exception( +- 'We can\'t read some files that are required to run the Magento application. ' +- . 'This usually means file permissions are set incorrectly.' +- ); +-} +- +-$vendorDir = require VENDOR_PATH; ++$vendorDir = './vendor'; + $vendorAutoload = BP . "/{$vendorDir}/autoload.php"; + + /* 'composer install' validation */ + +diff -Naur a/vendor/magento/framework/App/Arguments/FileResolver/Primary.php b/vendor/magento/framework/App/Arguments/FileResolver/Primary.php +index 40b74e9..0f732c9 100644 +--- a/vendor/magento/framework/App/Arguments/FileResolver/Primary.php ++++ b/vendor/magento/framework/App/Arguments/FileResolver/Primary.php +@@ -29,7 +29,7 @@ class Primary implements \Magento\Framework\Config\FileResolverInterface + \Magento\Framework\Filesystem $filesystem, + \Magento\Framework\Config\FileIteratorFactory $iteratorFactory + ) { +- $this->configDirectory = $filesystem->getDirectoryRead(DirectoryList::CONFIG); ++ $this->configDirectory = $filesystem->getDirectoryRead(DirectoryList::APP); + $this->iteratorFactory = $iteratorFactory; + } + diff --git a/patches/MAGETWO-57414__load_static_assets_without_rewrites__2.1.17.patch b/patches/MAGETWO-57414__load_static_assets_without_rewrites__2.1.17.patch new file mode 100644 index 0000000..e397a0d --- /dev/null +++ b/patches/MAGETWO-57414__load_static_assets_without_rewrites__2.1.17.patch @@ -0,0 +1,58 @@ +Ticket MAGETWO-57414 +diff -Naur a/vendor/magento/framework/App/StaticResource.php b/vendor/magento/framework/App/StaticResource.php +index d591deb..6344322 100644 +--- a/vendor/magento/framework/App/StaticResource.php ++++ b/vendor/magento/framework/App/StaticResource.php +@@ -94,24 +94,40 @@ class StaticResource implements \Magento\Framework\AppInterface + { + // disabling profiling when retrieving static resource + \Magento\Framework\Profiler::reset(); +- $appMode = $this->state->getMode(); +- if ($appMode == \Magento\Framework\App\State::MODE_PRODUCTION) { ++ $path = $this->getResourcePath(); ++ if (!isset($path)) { + $this->response->setHttpResponseCode(404); +- } else { +- $path = $this->request->get('resource'); +- $params = $this->parsePath($path); +- $this->state->setAreaCode($params['area']); +- $this->objectManager->configure($this->configLoader->load($params['area'])); +- $file = $params['file']; +- unset($params['file']); +- $asset = $this->assetRepo->createAsset($file, $params); +- $this->response->setFilePath($asset->getSourceFile()); +- $this->publisher->publish($asset); ++ return $this->response; + } ++ ++ $params = $this->parsePath($path); ++ $this->state->setAreaCode($params['area']); ++ $this->objectManager->configure($this->configLoader->load($params['area'])); ++ $file = $params['file']; ++ unset($params['file']); ++ $asset = $this->assetRepo->createAsset($file, $params); ++ $this->response->setFilePath($asset->getSourceFile()); ++ $this->publisher->publish($asset); + return $this->response; + } + + /** ++ * Retrieve the path from either the GET parameter or the request ++ * URI, depending on whether webserver rewrites are in use. ++ */ ++ protected function getResourcePath() { ++ $path = $this->request->get('resource'); ++ if (isset($path)) { ++ return $path; ++ } ++ ++ $path = $this->request->getUri()->getPath(); ++ if (preg_match("~^/static/(?:version\d*/)?(.*)$~", $path, $matches)) { ++ return $matches[1]; ++ } ++ } ++ ++ /** + * @inheritdoc + */ + public function catchException(Bootstrap $bootstrap, \Exception $exception) diff --git a/patches/MAGETWO-57414__load_static_assets_without_rewrites__2.1.4.patch b/patches/MAGETWO-57414__load_static_assets_without_rewrites__2.1.4.patch new file mode 100644 index 0000000..d1d93cc --- /dev/null +++ b/patches/MAGETWO-57414__load_static_assets_without_rewrites__2.1.4.patch @@ -0,0 +1,58 @@ +Ticket MAGETWO-57414 +diff -Naur a/vendor/magento/framework/App/StaticResource.php b/vendor/magento/framework/App/StaticResource.php +index d591deb..6344322 100644 +--- a/vendor/magento/framework/App/StaticResource.php ++++ b/vendor/magento/framework/App/StaticResource.php +@@ -94,24 +94,40 @@ class StaticResource implements \Magento\Framework\AppInterface + { + // disabling profiling when retrieving static resource + \Magento\Framework\Profiler::reset(); +- $appMode = $this->state->getMode(); +- if ($appMode == \Magento\Framework\App\State::MODE_PRODUCTION) { ++ $path = $this->getResourcePath(); ++ if (!isset($path)) { + $this->response->setHttpResponseCode(404); +- } else { +- $path = $this->request->get('resource'); +- $params = $this->parsePath($path); +- $this->state->setAreaCode($params['area']); +- $this->objectManager->configure($this->configLoader->load($params['area'])); +- $file = $params['file']; +- unset($params['file']); +- $asset = $this->assetRepo->createAsset($file, $params); +- $this->response->setFilePath($asset->getSourceFile()); +- $this->publisher->publish($asset); ++ return $this->response; + } ++ ++ $params = $this->parsePath($path); ++ $this->state->setAreaCode($params['area']); ++ $this->objectManager->configure($this->configLoader->load($params['area'])); ++ $file = $params['file']; ++ unset($params['file']); ++ $asset = $this->assetRepo->createAsset($file, $params); ++ $this->response->setFilePath($asset->getSourceFile()); ++ $this->publisher->publish($asset); + return $this->response; + } + + /** ++ * Retrieve the path from either the GET parameter or the request ++ * URI, depending on whether webserver rewrites are in use. ++ */ ++ protected function getResourcePath() { ++ $path = $this->request->get('resource'); ++ if (isset($path)) { ++ return $path; ++ } ++ ++ $path = $this->request->getUri()->getPath(); ++ if (preg_match("~^/static/(?:version\d*/)?(.*)$~", $path, $matches)) { ++ return $matches[1]; ++ } ++ } ++ ++ /** + * {@inheritdoc} + */ + public function catchException(Bootstrap $bootstrap, \Exception $exception) diff --git a/patches/MAGETWO-62660__prevent_excessive_js_optimization__2.1.4.patch b/patches/MAGETWO-62660__prevent_excessive_js_optimization__2.1.4.patch new file mode 100644 index 0000000..856c452 --- /dev/null +++ b/patches/MAGETWO-62660__prevent_excessive_js_optimization__2.1.4.patch @@ -0,0 +1,521 @@ +diff --git a/vendor/magento/module-deploy/Model/Deploy/JsDictionaryDeploy.php b/vendor/magento/module-deploy/Model/Deploy/JsDictionaryDeploy.php +new file mode 100644 +index 0000000..9c6a309a +--- /dev/null ++++ b/vendor/magento/module-deploy/Model/Deploy/JsDictionaryDeploy.php +@@ -0,0 +1,89 @@ ++assetRepo = $assetRepo; ++ $this->assetPublisher = $assetPublisher; ++ $this->translationJsConfig = $translationJsConfig; ++ $this->translator = $translator; ++ $this->output = $output; ++ } ++ ++ /** ++ * {@inheritdoc} ++ */ ++ public function deploy($area, $themePath, $locale) ++ { ++ $this->translator->setLocale($locale); ++ $this->translator->loadData($area, true); ++ ++ $asset = $this->assetRepo->createAsset( ++ $this->translationJsConfig->getDictionaryFileName(), ++ ['area' => $area, 'theme' => $themePath, 'locale' => $locale] ++ ); ++ if ($this->output->isVeryVerbose()) { ++ $this->output->writeln("\tDeploying the file to '{$asset->getPath()}'"); ++ } else { ++ $this->output->write('.'); ++ } ++ ++ $this->assetPublisher->publish($asset); ++ ++ return Cli::RETURN_SUCCESS; ++ } ++} +diff --git a/vendor/magento/module-deploy/Model/Deploy/LocaleQuickDeploy.php b/vendor/magento/module-deploy/Model/Deploy/LocaleQuickDeploy.php +index 0d990b5..aa23833 100644 +--- a/vendor/magento/module-deploy/Model/Deploy/LocaleQuickDeploy.php ++++ b/vendor/magento/module-deploy/Model/Deploy/LocaleQuickDeploy.php +@@ -6,16 +6,21 @@ + + namespace Magento\Deploy\Model\Deploy; + +-use Magento\Deploy\Model\DeployManager; + use Magento\Framework\App\Filesystem\DirectoryList; +-use Magento\Framework\App\Utility\Files; + use Magento\Framework\Filesystem; + use Magento\Framework\Filesystem\Directory\WriteInterface; + use Symfony\Component\Console\Output\OutputInterface; + use Magento\Framework\Console\Cli; + use Magento\Deploy\Console\Command\DeployStaticOptionsInterface as Options; +-use \Magento\Framework\RequireJs\Config as RequireJsConfig; ++use Magento\Framework\RequireJs\Config as RequireJsConfig; ++use Magento\Framework\Translate\Js\Config as TranslationJsConfig; ++use Magento\Framework\App\ObjectManager; ++use Magento\Deploy\Model\DeployStrategyFactory; + ++/** ++ * To avoid duplication of deploying of all static content per each theme/local, this class uses copying/symlinking ++ * of default static files to other locales, separately calls deploy for js dictionary per each locale ++ */ + class LocaleQuickDeploy implements DeployInterface + { + /** +@@ -39,15 +44,41 @@ class LocaleQuickDeploy implements DeployInterface + private $options = []; + + /** ++ * @var TranslationJsConfig ++ */ ++ private $translationJsConfig; ++ ++ /** ++ * @var DeployStrategyFactory ++ */ ++ private $deployStrategyFactory; ++ ++ /** ++ * @var DeployInterface[] ++ */ ++ private $deploys; ++ ++ /** + * @param Filesystem $filesystem + * @param OutputInterface $output + * @param array $options ++ * @param TranslationJsConfig $translationJsConfig ++ * @param DeployStrategyFactory $deployStrategyFactory + */ +- public function __construct(\Magento\Framework\Filesystem $filesystem, OutputInterface $output, $options = []) +- { ++ public function __construct( ++ Filesystem $filesystem, ++ OutputInterface $output, ++ $options = [], ++ TranslationJsConfig $translationJsConfig = null, ++ DeployStrategyFactory $deployStrategyFactory = null ++ ) { + $this->filesystem = $filesystem; + $this->output = $output; + $this->options = $options; ++ $this->translationJsConfig = $translationJsConfig ++ ?: ObjectManager::getInstance()->get(TranslationJsConfig::class); ++ $this->deployStrategyFactory = $deployStrategyFactory ++ ?: ObjectManager::getInstance()->get(DeployStrategyFactory::class); + } + + /** +@@ -67,13 +98,13 @@ private function getStaticDirectory() + */ + public function deploy($area, $themePath, $locale) + { +- if (isset($this->options[Options::DRY_RUN]) && $this->options[Options::DRY_RUN]) { ++ if (!empty($this->options[Options::DRY_RUN])) { + return Cli::RETURN_SUCCESS; + } + + $this->output->writeln("=== {$area} -> {$themePath} -> {$locale} ==="); + +- if (!isset($this->options[self::DEPLOY_BASE_LOCALE])) { ++ if (empty($this->options[self::DEPLOY_BASE_LOCALE])) { + throw new \InvalidArgumentException('Deploy base locale must be set for Quick Deploy'); + } + $processedFiles = 0; +@@ -88,7 +119,7 @@ public function deploy($area, $themePath, $locale) + $this->deleteLocaleResource($newLocalePath); + $this->deleteLocaleResource($newRequireJsPath); + +- if (isset($this->options[Options::SYMLINK_LOCALE]) && $this->options[Options::SYMLINK_LOCALE]) { ++ if (!empty($this->options[Options::SYMLINK_LOCALE])) { + $this->getStaticDirectory()->createSymlink($baseLocalePath, $newLocalePath); + $this->getStaticDirectory()->createSymlink($baseRequireJsPath, $newRequireJsPath); + +@@ -98,14 +129,29 @@ public function deploy($area, $themePath, $locale) + $this->getStaticDirectory()->readRecursively($baseLocalePath), + $this->getStaticDirectory()->readRecursively($baseRequireJsPath) + ); ++ $jsDictionaryEnabled = $this->translationJsConfig->dictionaryEnabled(); + foreach ($localeFiles as $path) { + if ($this->getStaticDirectory()->isFile($path)) { +- $destination = $this->replaceLocaleInPath($path, $baseLocale, $locale); +- $this->getStaticDirectory()->copyFile($path, $destination); +- $processedFiles++; ++ if (!$jsDictionaryEnabled || !$this->isJsDictionary($path)) { ++ $destination = $this->replaceLocaleInPath($path, $baseLocale, $locale); ++ $this->getStaticDirectory()->copyFile($path, $destination); ++ $processedFiles++; ++ } + } + } + ++ if ($jsDictionaryEnabled) { ++ $this->getDeploy( ++ DeployStrategyFactory::DEPLOY_STRATEGY_JS_DICTIONARY, ++ [ ++ 'output' => $this->output, ++ 'translationJsConfig' => $this->translationJsConfig ++ ] ++ ) ++ ->deploy($area, $themePath, $locale); ++ $processedFiles++; ++ } ++ + $this->output->writeln("\nSuccessful copied: {$processedFiles} files; errors: {$errorAmount}\n---\n"); + } + +@@ -113,6 +159,32 @@ public function deploy($area, $themePath, $locale) + } + + /** ++ * Get deploy strategy according to required strategy ++ * ++ * @param string $strategy ++ * @param array $params ++ * @return DeployInterface ++ */ ++ private function getDeploy($strategy, $params) ++ { ++ if (empty($this->deploys[$strategy])) { ++ $this->deploys[$strategy] = $this->deployStrategyFactory->create($strategy, $params); ++ } ++ return $this->deploys[$strategy]; ++ } ++ ++ /** ++ * Define if provided path is js dictionary ++ * ++ * @param string $path ++ * @return bool ++ */ ++ private function isJsDictionary($path) ++ { ++ return strpos($path, $this->translationJsConfig->getDictionaryFileName()) !== false; ++ } ++ ++ /** + * @param string $path + * @return void + */ +diff --git a/vendor/magento/module-deploy/Model/DeployStrategyFactory.php b/vendor/magento/module-deploy/Model/DeployStrategyFactory.php +index 536f344..7ba159b 100644 +--- a/vendor/magento/module-deploy/Model/DeployStrategyFactory.php ++++ b/vendor/magento/module-deploy/Model/DeployStrategyFactory.php +@@ -23,6 +23,11 @@ class DeployStrategyFactory + const DEPLOY_STRATEGY_QUICK = 'quick'; + + /** ++ * Strategy for deploying js dictionary ++ */ ++ const DEPLOY_STRATEGY_JS_DICTIONARY = 'js-dictionary'; ++ ++ /** + * @param ObjectManagerInterface $objectManager + */ + public function __construct(ObjectManagerInterface $objectManager) +@@ -41,6 +46,7 @@ public function create($type, array $arguments = []) + $strategyMap = [ + self::DEPLOY_STRATEGY_STANDARD => Deploy\LocaleDeploy::class, + self::DEPLOY_STRATEGY_QUICK => Deploy\LocaleQuickDeploy::class, ++ self::DEPLOY_STRATEGY_JS_DICTIONARY => Deploy\JsDictionaryDeploy::class + ]; + + if (!isset($strategyMap[$type])) { +diff --git a/vendor/magento/module-deploy/Test/Unit/Model/Deploy/JsDictionaryDeployTest.php b/vendor/magento/module-deploy/Test/Unit/Model/Deploy/JsDictionaryDeployTest.php +new file mode 100644 +index 0000000..2533476 +--- /dev/null ++++ b/vendor/magento/module-deploy/Test/Unit/Model/Deploy/JsDictionaryDeployTest.php +@@ -0,0 +1,103 @@ ++output = $this->getMockBuilder(OutputInterface::class) ++ ->setMethods(['writeln', 'isVeryVerbose']) ++ ->getMockForAbstractClass(); ++ ++ $this->translationJsConfig = $this->getMock(TranslationJsConfig::class, [], [], '', false); ++ $this->translator = $this->getMockForAbstractClass(TranslateInterface::class, [], '', false, false, true); ++ $this->assetRepo = $this->getMock(Repository::class, [], [], '', false); ++ $this->asset = $this->getMockForAbstractClass(Asset::class, [], '', false, false, true); ++ $this->assetPublisher = $this->getMock(Publisher::class, [], [], '', false); ++ ++ $this->model = (new ObjectManager($this))->getObject( ++ JsDictionaryDeploy::class, ++ [ ++ 'translationJsConfig' => $this->translationJsConfig, ++ 'translator' => $this->translator, ++ 'assetRepo' => $this->assetRepo, ++ 'assetPublisher' => $this->assetPublisher, ++ 'output' => $this->output ++ ] ++ ); ++ } ++ ++ public function testDeploy() ++ { ++ $area = 'adminhtml'; ++ $themePath = 'Magento/backend'; ++ $locale = 'uk_UA'; ++ ++ $dictionary = 'js-translation.json'; ++ ++ $this->translationJsConfig->expects(self::once())->method('getDictionaryFileName') ++ ->willReturn($dictionary); ++ ++ $this->translator->expects($this->once())->method('setLocale')->with($locale); ++ $this->translator->expects($this->once())->method('loadData')->with($area, true); ++ ++ $this->assetRepo->expects($this->once())->method('createAsset') ++ ->with( ++ $dictionary, ++ ['area' => $area, 'theme' => $themePath, 'locale' => $locale] ++ ) ++ ->willReturn($this->asset); ++ ++ $this->assetPublisher->expects($this->once())->method('publish'); ++ ++ $this->model->deploy($area, $themePath, $locale); ++ } ++} +diff --git a/vendor/magento/module-deploy/Test/Unit/Model/Deploy/LocaleQuickDeployTest.php b/vendor/magento/module-deploy/Test/Unit/Model/Deploy/LocaleQuickDeployTest.php +index 6c693fe..d50c8ce 100644 +--- a/vendor/magento/module-deploy/Test/Unit/Model/Deploy/LocaleQuickDeployTest.php ++++ b/vendor/magento/module-deploy/Test/Unit/Model/Deploy/LocaleQuickDeployTest.php +@@ -11,7 +11,10 @@ + use Magento\Framework\TestFramework\Unit\Helper\ObjectManager; + use Symfony\Component\Console\Output\OutputInterface; + use Magento\Deploy\Console\Command\DeployStaticOptionsInterface as Options; +-use \Magento\Framework\RequireJs\Config as RequireJsConfig; ++use Magento\Framework\RequireJs\Config as RequireJsConfig; ++use Magento\Framework\Translate\Js\Config as TranslationJsConfig; ++use Magento\Deploy\Model\Deploy\JsDictionaryDeploy; ++use Magento\Deploy\Model\DeployStrategyFactory; + + class LocaleQuickDeployTest extends \PHPUnit_Framework_TestCase + { +@@ -25,15 +28,32 @@ class LocaleQuickDeployTest extends \PHPUnit_Framework_TestCase + */ + private $staticDirectoryMock; + ++ /** ++ * @var TranslationJsConfig|\PHPUnit_Framework_MockObject_MockObject ++ */ ++ private $translationJsConfig; ++ ++ /** ++ * @var JsDictionaryDeploy|\PHPUnit_Framework_MockObject_MockObject ++ */ ++ private $jsDictionaryDeploy; ++ ++ /** ++ * @var DeployStrategyFactory|\PHPUnit_Framework_MockObject_MockObject ++ */ ++ private $deployStrategyFactory; ++ + protected function setUp() + { + $this->outputMock = $this->getMockBuilder(OutputInterface::class) +- ->setMethods(['writeln']) ++ ->setMethods(['writeln', 'isVeryVerbose']) + ->getMockForAbstractClass(); +- + $this->staticDirectoryMock = $this->getMockBuilder(WriteInterface::class) + ->setMethods(['createSymlink', 'getAbsolutePath', 'getRelativePath', 'copyFile', 'readRecursively']) + ->getMockForAbstractClass(); ++ $this->translationJsConfig = $this->getMock(TranslationJsConfig::class, [], [], '', false); ++ $this->deployStrategyFactory = $this->getMock(DeployStrategyFactory::class, [], [], '', false); ++ $this->jsDictionaryDeploy = $this->getMock(JsDictionaryDeploy::class, [], [], '', false); + } + + /** +@@ -68,29 +88,53 @@ public function testDeployWithSymlinkStrategy() + + public function testDeployWithCopyStrategy() + { +- + $area = 'adminhtml'; + $themePath = 'Magento/backend'; + $locale = 'uk_UA'; +- $baseLocal = 'en_US'; ++ $baseLocale = 'en_US'; ++ $baseDir = $baseLocale . 'dir'; ++ $file1 = 'file1'; ++ $file2 = 'file2'; ++ $baseFile1 = $baseLocale . $file1; ++ $baseFile2 = $baseLocale . $file2; ++ ++ $dictionary = 'js-translation.json'; ++ $baseDictionary = $baseLocale . $dictionary; + + $this->staticDirectoryMock->expects(self::never())->method('createSymlink'); +- $this->staticDirectoryMock->expects(self::exactly(2))->method('readRecursively')->willReturnMap([ +- ['adminhtml/Magento/backend/en_US', [$baseLocal . 'file1', $baseLocal . 'dir']], +- [RequireJsConfig::DIR_NAME . '/adminhtml/Magento/backend/en_US', [$baseLocal . 'file2']] +- ]); +- $this->staticDirectoryMock->expects(self::exactly(3))->method('isFile')->willReturnMap([ +- [$baseLocal . 'file1', true], +- [$baseLocal . 'dir', false], +- [$baseLocal . 'file2', true], ++ $this->staticDirectoryMock->expects(self::exactly(2))->method('readRecursively')->willReturnMap( ++ [ ++ ['adminhtml/Magento/backend/en_US', [$baseFile1, $baseDir]], ++ [RequireJsConfig::DIR_NAME . '/adminhtml/Magento/backend/en_US', [$baseFile2, $baseDictionary]] ++ ] ++ ); ++ $this->staticDirectoryMock->expects(self::exactly(4))->method('isFile')->willReturnMap([ ++ [$baseFile1, true], ++ [$baseDir, false], ++ [$baseFile2, true], ++ [$baseDictionary, true] + ]); + $this->staticDirectoryMock->expects(self::exactly(2))->method('copyFile')->withConsecutive( +- [$baseLocal . 'file1', $locale . 'file1', null], +- [$baseLocal . 'file2', $locale . 'file2', null] ++ [$baseFile1, $locale . $file1, null], ++ [$baseFile2, $locale . $file2, null] + ); + ++ $this->translationJsConfig->expects(self::exactly(3))->method('getDictionaryFileName') ++ ->willReturn($dictionary); ++ ++ $this->translationJsConfig->expects($this->once())->method('dictionaryEnabled')->willReturn(true); ++ ++ $this->deployStrategyFactory->expects($this->once())->method('create') ++ ->with( ++ DeployStrategyFactory::DEPLOY_STRATEGY_JS_DICTIONARY, ++ ['output' => $this->outputMock, 'translationJsConfig' => $this->translationJsConfig] ++ ) ++ ->willReturn($this->jsDictionaryDeploy); ++ ++ $this->jsDictionaryDeploy->expects($this->once())->method('deploy')->with($area, $themePath, $locale); ++ + $model = $this->getModel([ +- DeployInterface::DEPLOY_BASE_LOCALE => $baseLocal, ++ DeployInterface::DEPLOY_BASE_LOCALE => $baseLocale, + Options::SYMLINK_LOCALE => 0, + ]); + $model->deploy($area, $themePath, $locale); +@@ -107,7 +151,9 @@ private function getModel($options = []) + [ + 'output' => $this->outputMock, + 'staticDirectory' => $this->staticDirectoryMock, +- 'options' => $options ++ 'options' => $options, ++ 'translationJsConfig' => $this->translationJsConfig, ++ 'deployStrategyFactory' => $this->deployStrategyFactory + ] + ); + } diff --git a/patches/MAGETWO-63020__fix_scd_with_multiple_languages__2.1.4.patch b/patches/MAGETWO-63020__fix_scd_with_multiple_languages__2.1.4.patch new file mode 100644 index 0000000..a291f59 --- /dev/null +++ b/patches/MAGETWO-63020__fix_scd_with_multiple_languages__2.1.4.patch @@ -0,0 +1,13 @@ +MAGETWO-63020 +diff --Naur a/vendor/magento/framework/View/Design/Theme/FlyweightFactory.php b/vendor/magento/framework/View/Design/Theme/FlyweightFactory.php +--- a/vendor/magento/framework/View/Design/Theme/FlyweightFactory.php 2017-01-06 20:41:07.000000000 +0000 ++++ b/vendor/magento/framework/View/Design/Theme/FlyweightFactory.php 2017-01-06 20:42:33.000000000 +0000 +@@ -61,7 +61,7 @@ + } else { + $themeModel = $this->_loadByPath($themeKey, $area); + } +- if (!$themeModel->getId()) { ++ if (!$themeModel->getCode()) { + throw new \LogicException("Unable to load theme by specified key: '{$themeKey}'"); + } + $this->_addTheme($themeModel); diff --git a/patches/MAGETWO-63032__skip_unnecessary_write_permission_check__2.1.4.patch b/patches/MAGETWO-63032__skip_unnecessary_write_permission_check__2.1.4.patch new file mode 100644 index 0000000..67c7fb2 --- /dev/null +++ b/patches/MAGETWO-63032__skip_unnecessary_write_permission_check__2.1.4.patch @@ -0,0 +1,20 @@ +MAGETWO-63032 +diff -Naur a/vendor/magento/framework/Console/Cli.php b/vendor/magento/framework/Console/Cli.php +--- a/vendor/magento/framework/Console/Cli.php 2016-09-28 00:46:02.000000000 -0500 ++++ b/vendor/magento/framework/Console/Cli.php 2016-09-28 00:46:38.000000000 -0500 +@@ -56,15 +56,6 @@ + { + $this->serviceManager = \Zend\Mvc\Application::init(require BP . '/setup/config/application.config.php') + ->getServiceManager(); +- $generationDirectoryAccess = new GenerationDirectoryAccess($this->serviceManager); +- if (!$generationDirectoryAccess->check()) { +- $output = new ConsoleOutput(); +- $output->writeln( +- 'Command line user does not have read and write permissions on var/generation directory. Please' +- . ' address this issue before using Magento command line.' +- ); +- exit(0); +- } + /** + * Temporary workaround until the compiler is able to clear the generation directory + * @todo remove after MAGETWO-44493 resolved diff --git a/patches/MAGETWO-67097__fix_credis_pipeline_bug__2.1.4.patch b/patches/MAGETWO-67097__fix_credis_pipeline_bug__2.1.4.patch new file mode 100644 index 0000000..6b9b89b --- /dev/null +++ b/patches/MAGETWO-67097__fix_credis_pipeline_bug__2.1.4.patch @@ -0,0 +1,12 @@ +diff -Nuar a/vendor/colinmollenhour/credis/Client.php b/vendor/colinmollenhour/credis/Client.php +index afbc85d..8368b32 100755 +--- a/vendor/colinmollenhour/credis/Client.php ++++ b/vendor/colinmollenhour/credis/Client.php +@@ -1017,6 +1017,7 @@ class Credis_Client { + } else { + $this->isMulti = TRUE; + $this->redisMulti = call_user_func_array(array($this->redis, $name), $args); ++ return $this; + } + } + else if($name == 'exec' || $name == 'discard') { diff --git a/patches/MAGETWO-67805__fix_image_resizing_after_upgrade__2.1.6.patch b/patches/MAGETWO-67805__fix_image_resizing_after_upgrade__2.1.6.patch new file mode 100644 index 0000000..15b6451 --- /dev/null +++ b/patches/MAGETWO-67805__fix_image_resizing_after_upgrade__2.1.6.patch @@ -0,0 +1,187 @@ +diff -Nuar a/vendor/magento/module-catalog/Block/Product/ImageBlockBuilder.php b/vendor/magento/module-catalog/Block/Product/ImageBlockBuilder.php +index 9fa50c1..1e54cfc 100644 +--- a/vendor/magento/module-catalog/Block/Product/ImageBlockBuilder.php ++++ b/vendor/magento/module-catalog/Block/Product/ImageBlockBuilder.php +@@ -120,12 +120,7 @@ class ImageBlockBuilder + $label = $product->getName(); + } + +- $frame = isset($imageArguments['frame']) ? $imageArguments ['frame'] : null; +- if (empty($frame)) { +- $frame = $this->presentationConfig->getVarValue('Magento_Catalog', 'product_image_white_borders'); +- } +- +- $template = $frame ++ $template = $image['keep_frame'] == 'frame' + ? 'Magento_Catalog::product/image.phtml' + : 'Magento_Catalog::product/image_with_borders.phtml'; + +diff -Nuar a/vendor/magento/module-catalog/Helper/Image.php b/vendor/magento/module-catalog/Helper/Image.php +index 039c260..9a7c13e 100644 +--- a/vendor/magento/module-catalog/Helper/Image.php ++++ b/vendor/magento/module-catalog/Helper/Image.php +@@ -826,7 +826,7 @@ class Image extends AbstractHelper + if ($frame === null) { + $frame = $this->getConfigView()->getVarValue('Magento_Catalog', 'product_image_white_borders'); + } +- return (bool)$frame; ++ return $frame; + } + + /** +diff -Nuar a/vendor/magento/module-catalog/Model/Product/Image.php b/vendor/magento/module-catalog/Model/Product/Image.php +index c1c10da..b543571 100644 +--- a/vendor/magento/module-catalog/Model/Product/Image.php ++++ b/vendor/magento/module-catalog/Model/Product/Image.php +@@ -320,7 +320,7 @@ class Image extends \Magento\Framework\Model\AbstractModel + */ + public function setKeepAspectRatio($keep) + { +- $this->_keepAspectRatio = (bool)$keep; ++ $this->_keepAspectRatio = $keep && $keep !== 'false'; + return $this; + } + +@@ -330,7 +330,7 @@ class Image extends \Magento\Framework\Model\AbstractModel + */ + public function setKeepFrame($keep) + { +- $this->_keepFrame = (bool)$keep; ++ $this->_keepFrame = $keep && $keep !== 'false'; + return $this; + } + +@@ -340,7 +340,7 @@ class Image extends \Magento\Framework\Model\AbstractModel + */ + public function setKeepTransparency($keep) + { +- $this->_keepTransparency = (bool)$keep; ++ $this->_keepTransparency = $keep && $keep !== 'false'; + return $this; + } + +@@ -350,7 +350,7 @@ class Image extends \Magento\Framework\Model\AbstractModel + */ + public function setConstrainOnly($flag) + { +- $this->_constrainOnly = (bool)$flag; ++ $this->_constrainOnly = $flag && $flag !== 'false'; + return $this; + } + +diff -Nuar a/vendor/magento/module-catalog/Model/Product/Image/ParamsBuilder.php b/vendor/magento/module-catalog/Model/Product/Image/ParamsBuilder.php +index 603f3e8..7aa0c32 100644 +--- a/vendor/magento/module-catalog/Model/Product/Image/ParamsBuilder.php ++++ b/vendor/magento/module-catalog/Model/Product/Image/ParamsBuilder.php +@@ -70,6 +70,21 @@ class ParamsBuilder + } + + /** ++ * Reads boolean value from arguments ++ * ++ * Performs correct boolean cast ++ * ++ * @param array $args ++ * @param string $key ++ * @param bool $default ++ * @return bool|null ++ */ ++ private function readBoolValue(array $args, $key, $default = null) ++ { ++ return isset($args[$key]) ? $args[$key] && $args[$key] !== 'false' : $default; ++ } ++ ++ /** + * @param array $imageArguments + * @return array + * @SuppressWarnings(PHPMD.NPathComplexity) +@@ -82,9 +97,11 @@ class ParamsBuilder + $width = isset($imageArguments['width']) ? $imageArguments['width'] : null; + $height = isset($imageArguments['height']) ? $imageArguments['height'] : null; + +- $frame = !empty($imageArguments['frame']) +- ? $imageArguments['frame'] +- : $this->keepFrame; ++ $frame = $this->readBoolValue($imageArguments, 'frame'); ++ ++ if ($frame === null) { ++ $frame = $this->presentationConfig->getVarValue('Magento_Catalog', 'product_image_white_borders'); ++ } + + $constrain = !empty($imageArguments['constrain']) + ? $imageArguments['constrain'] +diff -Nuar a/vendor/magento/module-catalog/etc/view.xml b/vendor/magento/module-catalog/etc/view.xml +index 8c7500d..f2f004c 100644 +--- a/vendor/magento/module-catalog/etc/view.xml ++++ b/vendor/magento/module-catalog/etc/view.xml +@@ -7,6 +7,6 @@ + --> + + +- 1 ++ 0 + + +diff -Nuar a/vendor/magento/module-catalog/view/frontend/templates/product/image_with_borders.phtml b/vendor/magento/module-catalog/view/frontend/templates/product/image_with_borders.phtml +index c3280c5..4d7a825 100644 +--- a/vendor/magento/module-catalog/view/frontend/templates/product/image_with_borders.phtml ++++ b/vendor/magento/module-catalog/view/frontend/templates/product/image_with_borders.phtml +@@ -13,7 +13,5 @@ + getCustomAttributes(); ?> + src="getImageUrl(); ?>" +- width="getResizedImageWidth(); ?>" +- height="getResizedImageHeight(); ?>" + alt="stripTags($block->getLabel(), null, true); ?>"/> + +diff -Nuar a/vendor/magento/theme-adminhtml-backend/etc/view.xml b/vendor/magento/theme-adminhtml-backend/etc/view.xml +index de6b0cf..f352974 100644 +--- a/vendor/magento/theme-adminhtml-backend/etc/view.xml ++++ b/vendor/magento/theme-adminhtml-backend/etc/view.xml +@@ -9,6 +9,9 @@ + + 1MB + ++ ++ 0 ++ + + + +diff -Nuar a/lib/web/mage/gallery/gallery.less b/lib/web/mage/gallery/gallery.less +index fc9d09e..2fd877d 100644 +--- a/lib/web/mage/gallery/gallery.less ++++ b/lib/web/mage/gallery/gallery.less +@@ -691,23 +691,15 @@ + } + + .fotorama__nav-wrap { +- .fotorama_vertical_ratio { +- .fotorama__img { +- .translateY(-50%); +- height: auto; +- position: absolute; +- top: 50%; +- width: 100%; +- } +- } +- .fotorama_horizontal_ratio { +- .fotorama__img { +- .translateX(-50%); +- height: 100%; +- left: 50%; +- position: absolute; +- width: auto; +- } ++ .fotorama__img { ++ bottom: 0; ++ height: auto; ++ left: 0; ++ margin: auto; ++ max-width: 100%; ++ position: absolute; ++ right: 0; ++ top: 0; + } + } + diff --git a/patches/MAGETWO-69847__support_credis_forking_during_scd__2.1.4.patch b/patches/MAGETWO-69847__support_credis_forking_during_scd__2.1.4.patch new file mode 100644 index 0000000..fd1697e --- /dev/null +++ b/patches/MAGETWO-69847__support_credis_forking_during_scd__2.1.4.patch @@ -0,0 +1,169 @@ +diff -Nuar a/vendor/magento/framework/App/Cache/Frontend/Factory.php b/vendor/magento/framework/App/Cache/Frontend/Factory.php +index a71ff27b07..e1c65ef572 100644 +--- a/vendor/magento/framework/App/Cache/Frontend/Factory.php ++++ b/vendor/magento/framework/App/Cache/Frontend/Factory.php +@@ -145,15 +145,17 @@ class Factory + $result = $this->_objectManager->create( + 'Magento\Framework\Cache\Frontend\Adapter\Zend', + [ +- 'frontend' => \Zend_Cache::factory( +- $frontend['type'], +- $backend['type'], +- $frontend, +- $backend['options'], +- true, +- true, +- true +- ) ++ 'frontendFactory' => function () use ($frontend, $backend) { ++ return \Zend_Cache::factory( ++ $frontend['type'], ++ $backend['type'], ++ $frontend, ++ $backend['options'], ++ true, ++ true, ++ true ++ ); ++ } + ] + ); + $result = $this->_applyDecorators($result); +diff -Nuar a/vendor/magento/framework/Cache/Frontend/Adapter/Zend.php b/vendor/magento/framework/Cache/Frontend/Adapter/Zend.php +index 5d72ee6a1e..fe9dc2f453 100644 +--- a/vendor/magento/framework/Cache/Frontend/Adapter/Zend.php ++++ b/vendor/magento/framework/Cache/Frontend/Adapter/Zend.php +@@ -16,11 +16,32 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + protected $_frontend; + + /** +- * @param \Zend_Cache_Core $frontend ++ * Factory that creates the \Zend_Cache_Cores ++ * ++ * @var \Closure ++ */ ++ private $_frontendFactory; ++ ++ /** ++ * The pid that owns the $_frontend object ++ * ++ * @var int ++ */ ++ private $_pid; ++ ++ /** ++ * @var \Zend_Cache_Core[] ++ */ ++ static private $_parentFrontends = []; ++ ++ /** ++ * @param \Closure $frontendFactory + */ +- public function __construct(\Zend_Cache_Core $frontend) ++ public function __construct(\Closure $frontendFactory) + { +- $this->_frontend = $frontend; ++ $this->_frontendFactory = $frontendFactory; ++ $this->_frontend = $frontendFactory(); ++ $this->_pid = getmypid(); + } + + /** +@@ -28,7 +49,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function test($identifier) + { +- return $this->_frontend->test($this->_unifyId($identifier)); ++ return $this->_getFrontEnd()->test($this->_unifyId($identifier)); + } + + /** +@@ -36,7 +57,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function load($identifier) + { +- return $this->_frontend->load($this->_unifyId($identifier)); ++ return $this->_getFrontEnd()->load($this->_unifyId($identifier)); + } + + /** +@@ -44,7 +65,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function save($data, $identifier, array $tags = [], $lifeTime = null) + { +- return $this->_frontend->save($data, $this->_unifyId($identifier), $this->_unifyIds($tags), $lifeTime); ++ return $this->_getFrontEnd()->save($data, $this->_unifyId($identifier), $this->_unifyIds($tags), $lifeTime); + } + + /** +@@ -52,7 +73,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function remove($identifier) + { +- return $this->_frontend->remove($this->_unifyId($identifier)); ++ return $this->_getFrontEnd()->remove($this->_unifyId($identifier)); + } + + /** +@@ -76,7 +97,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + "Magento cache frontend does not support the cleaning mode '{$mode}'." + ); + } +- return $this->_frontend->clean($mode, $this->_unifyIds($tags)); ++ return $this->_getFrontEnd()->clean($mode, $this->_unifyIds($tags)); + } + + /** +@@ -84,7 +105,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function getBackend() + { +- return $this->_frontend->getBackend(); ++ return $this->_getFrontEnd()->getBackend(); + } + + /** +@@ -92,7 +113,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function getLowLevelFrontend() + { +- return $this->_frontend; ++ return $this->_getFrontEnd(); + } + + /** +@@ -119,4 +140,34 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + } + return $ids; + } ++ ++ /** ++ * getter for _frontend so that we can support fork()s ++ * ++ * @return \Zend_Cache_Core ++ */ ++ private function _getFrontEnd() ++ { ++ if (getmypid() === $this->_pid) { ++ return $this->_frontend; ++ } ++ static::$_parentFrontends[] = $this->_frontend; ++ $frontendFactory = $this->_frontendFactory; ++ $this->_frontend = $frontendFactory(); ++ $this->_pid = getmypid(); ++ return $this->_frontend; ++ } ++ ++ /** ++ * If the current _frontend is owned by a different pid, add it to $_parentFrontends so that the ++ * destructor isn't called on it. ++ * ++ * @return void ++ */ ++ public function __destruct() ++ { ++ if (getmypid() !== $this->_pid) { ++ static::$_parentFrontends[] = $this->_frontend; ++ } ++ } + } diff --git a/patches/MAGETWO-69847__support_credis_forking_during_scd__2.2.0.patch b/patches/MAGETWO-69847__support_credis_forking_during_scd__2.2.0.patch new file mode 100644 index 0000000..8c994b2 --- /dev/null +++ b/patches/MAGETWO-69847__support_credis_forking_during_scd__2.2.0.patch @@ -0,0 +1,169 @@ +diff -Nuar a/vendor/magento/framework/App/Cache/Frontend/Factory.php b/vendor/magento/framework/App/Cache/Frontend/Factory.php +index 4c539da096..8477d94f28 100644 +--- a/vendor/magento/framework/App/Cache/Frontend/Factory.php ++++ b/vendor/magento/framework/App/Cache/Frontend/Factory.php +@@ -148,15 +148,17 @@ class Factory + $result = $this->_objectManager->create( + \Magento\Framework\Cache\Frontend\Adapter\Zend::class, + [ +- 'frontend' => \Zend_Cache::factory( +- $frontend['type'], +- $backend['type'], +- $frontend, +- $backend['options'], +- true, +- true, +- true +- ) ++ 'frontendFactory' => function () use ($frontend, $backend) { ++ return \Zend_Cache::factory( ++ $frontend['type'], ++ $backend['type'], ++ $frontend, ++ $backend['options'], ++ true, ++ true, ++ true ++ ); ++ } + ] + ); + $result = $this->_applyDecorators($result); +diff -Nuar a/vendor/magento/framework/Cache/Frontend/Adapter/Zend.php b/vendor/magento/framework/Cache/Frontend/Adapter/Zend.php +index c8917a0996..fe9dc2f453 100644 +--- a/vendor/magento/framework/Cache/Frontend/Adapter/Zend.php ++++ b/vendor/magento/framework/Cache/Frontend/Adapter/Zend.php +@@ -16,11 +16,32 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + protected $_frontend; + + /** +- * @param \Zend_Cache_Core $frontend ++ * Factory that creates the \Zend_Cache_Cores ++ * ++ * @var \Closure ++ */ ++ private $_frontendFactory; ++ ++ /** ++ * The pid that owns the $_frontend object ++ * ++ * @var int ++ */ ++ private $_pid; ++ ++ /** ++ * @var \Zend_Cache_Core[] ++ */ ++ static private $_parentFrontends = []; ++ ++ /** ++ * @param \Closure $frontendFactory + */ +- public function __construct(\Zend_Cache_Core $frontend) ++ public function __construct(\Closure $frontendFactory) + { +- $this->_frontend = $frontend; ++ $this->_frontendFactory = $frontendFactory; ++ $this->_frontend = $frontendFactory(); ++ $this->_pid = getmypid(); + } + + /** +@@ -28,7 +49,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function test($identifier) + { +- return $this->_frontend->test($this->_unifyId($identifier)); ++ return $this->_getFrontEnd()->test($this->_unifyId($identifier)); + } + + /** +@@ -36,7 +57,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function load($identifier) + { +- return $this->_frontend->load($this->_unifyId($identifier)); ++ return $this->_getFrontEnd()->load($this->_unifyId($identifier)); + } + + /** +@@ -44,7 +65,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function save($data, $identifier, array $tags = [], $lifeTime = null) + { +- return $this->_frontend->save($data, $this->_unifyId($identifier), $this->_unifyIds($tags), $lifeTime); ++ return $this->_getFrontEnd()->save($data, $this->_unifyId($identifier), $this->_unifyIds($tags), $lifeTime); + } + + /** +@@ -52,7 +73,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function remove($identifier) + { +- return $this->_frontend->remove($this->_unifyId($identifier)); ++ return $this->_getFrontEnd()->remove($this->_unifyId($identifier)); + } + + /** +@@ -76,7 +97,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + "Magento cache frontend does not support the cleaning mode '{$mode}'." + ); + } +- return $this->_frontend->clean($mode, $this->_unifyIds($tags)); ++ return $this->_getFrontEnd()->clean($mode, $this->_unifyIds($tags)); + } + + /** +@@ -84,7 +105,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function getBackend() + { +- return $this->_frontend->getBackend(); ++ return $this->_getFrontEnd()->getBackend(); + } + + /** +@@ -92,7 +113,7 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + */ + public function getLowLevelFrontend() + { +- return $this->_frontend; ++ return $this->_getFrontEnd(); + } + + /** +@@ -119,4 +140,34 @@ class Zend implements \Magento\Framework\Cache\FrontendInterface + } + return $ids; + } ++ ++ /** ++ * getter for _frontend so that we can support fork()s ++ * ++ * @return \Zend_Cache_Core ++ */ ++ private function _getFrontEnd() ++ { ++ if (getmypid() === $this->_pid) { ++ return $this->_frontend; ++ } ++ static::$_parentFrontends[] = $this->_frontend; ++ $frontendFactory = $this->_frontendFactory; ++ $this->_frontend = $frontendFactory(); ++ $this->_pid = getmypid(); ++ return $this->_frontend; ++ } ++ ++ /** ++ * If the current _frontend is owned by a different pid, add it to $_parentFrontends so that the ++ * destructor isn't called on it. ++ * ++ * @return void ++ */ ++ public function __destruct() ++ { ++ if (getmypid() !== $this->_pid) { ++ static::$_parentFrontends[] = $this->_frontend; ++ } ++ } + } diff --git a/patches/MAGETWO-82752__reload_js_translation_data__2.2.0.patch b/patches/MAGETWO-82752__reload_js_translation_data__2.2.0.patch new file mode 100644 index 0000000..6eca987 --- /dev/null +++ b/patches/MAGETWO-82752__reload_js_translation_data__2.2.0.patch @@ -0,0 +1,11 @@ +--- a/vendor/magento/module-translation/Model/Json/PreProcessor.php ++++ b/vendor/magento/module-translation/Model/Json/PreProcessor.php +@@ -77,7 +77,7 @@ class PreProcessor implements PreProcessorInterface + if ($context instanceof FallbackContext) { + $themePath = $context->getThemePath(); + $areaCode = $context->getAreaCode(); +- $this->translate->setLocale($context->getLocale()); ++ $this->translate->setLocale($context->getLocale())->loadData($areaCode); + } + + $area = $this->areaList->getArea($areaCode); diff --git a/patches/MAGETWO-84444__fix_mview_on_staging__2.1.10.patch b/patches/MAGETWO-84444__fix_mview_on_staging__2.1.10.patch new file mode 100644 index 0000000..0b7b588 --- /dev/null +++ b/patches/MAGETWO-84444__fix_mview_on_staging__2.1.10.patch @@ -0,0 +1,1190 @@ +commit e9aa4e18cfb76b37ce00f6f45d507d22e1e89550 +Author: Viktor Paladiichuk +Date: Tue Nov 28 15:29:54 2017 +0200 + + MAGETWO-84444: Mview does not work with Staging + +diff -Nuar a/vendor/magento/module-catalog-inventory/etc/mview.xml b/vendor/magento/module-catalog-inventory/etc/mview.xml +index 58a051a3d0e..3dd8419d7e3 100644 +--- a/vendor/magento/module-catalog-inventory/etc/mview.xml ++++ b/vendor/magento/module-catalog-inventory/etc/mview.xml +@@ -5,10 +5,13 @@ + * See COPYING.txt for license details. + */ + --> +- ++ + + + ++
++
+ + + +diff -Nuar a/vendor/magento/module-indexer/Setup/RecurringData.php b/vendor/magento/module-indexer/Setup/RecurringData.php +new file mode 100644 +index 00000000000..38ea0e5b79e +--- /dev/null ++++ b/vendor/magento/module-indexer/Setup/RecurringData.php +@@ -0,0 +1,56 @@ ++indexerFactory = $indexerFactory; ++ $this->configInterface = $configInterface; ++ } ++ ++ /** ++ * {@inheritdoc} ++ */ ++ public function install(ModuleDataSetupInterface $setup, ModuleContextInterface $context) ++ { ++ foreach (array_keys($this->configInterface->getIndexers()) as $indexerId) { ++ $indexer = $this->indexerFactory->create()->load($indexerId); ++ if ($indexer->isScheduled()) { ++ $indexer->getView()->unsubscribe()->subscribe(); ++ } ++ } ++ } ++} +diff -Nuar a/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php b/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php +index aae82938c83..545689a4391 100644 +--- a/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php ++++ b/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php +@@ -62,7 +62,7 @@ class SubscriptionTest extends \PHPUnit_Framework_TestCase + + $this->resourceMock->expects($this->any()) + ->method('getTableName') +- ->willReturn($this->tableName); ++ ->will($this->returnArgument(0)); + + $this->model = new Subscription( + $this->resourceMock, +@@ -89,11 +89,15 @@ class SubscriptionTest extends \PHPUnit_Framework_TestCase + $this->assertEquals('columnName', $this->model->getColumnName()); + } + ++ /** ++ * @SuppressWarnings(PHPMD.ExcessiveMethodLength) ++ */ + public function testCreate() + { + $triggerName = 'trigger_name'; + $this->resourceMock->expects($this->atLeastOnce())->method('getTriggerName')->willReturn($triggerName); + $triggerMock = $this->getMockBuilder('Magento\Framework\DB\Ddl\Trigger') ++ ->setMethods(['setName', 'getName', 'setTime', 'setEvent', 'setTable', 'addStatement']) + ->disableOriginalConstructor() + ->getMock(); + $triggerMock->expects($this->exactly(3)) +@@ -114,8 +118,35 @@ class SubscriptionTest extends \PHPUnit_Framework_TestCase + ->method('setTable') + ->with($this->tableName) + ->will($this->returnSelf()); +- $triggerMock->expects($this->exactly(6)) ++ ++ $triggerMock->expects($this->at(4)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(5)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO other_test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(11)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(12)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO other_test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(18)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO test_view_cl (entity_id) VALUES (OLD.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(19)) + ->method('addStatement') ++ ->with("INSERT IGNORE INTO other_test_view_cl (entity_id) VALUES (OLD.columnName);") + ->will($this->returnSelf()); + + $changelogMock = $this->getMockForAbstractClass( +diff -Nuar a/vendor/magento/framework/Mview/View/Subscription.php b/vendor/magento/framework/Mview/View/Subscription.php +index c3da91c8331..3c4bd1dce2d 100644 +--- a/vendor/magento/framework/Mview/View/Subscription.php ++++ b/vendor/magento/framework/Mview/View/Subscription.php +@@ -10,6 +10,8 @@ namespace Magento\Framework\Mview\View; + + use Magento\Framework\App\ResourceConnection; + use Magento\Framework\DB\Ddl\Trigger; ++use Magento\Framework\DB\Ddl\TriggerFactory; ++use Magento\Framework\Mview\ViewInterface; + + class Subscription implements SubscriptionInterface + { +@@ -21,12 +23,12 @@ class Subscription implements SubscriptionInterface + protected $connection; + + /** +- * @var \Magento\Framework\DB\Ddl\TriggerFactory ++ * @var TriggerFactory + */ + protected $triggerFactory; + + /** +- * @var \Magento\Framework\Mview\View\CollectionInterface ++ * @var CollectionInterface + */ + protected $viewCollection; + +@@ -58,20 +60,31 @@ class Subscription implements SubscriptionInterface + protected $resource; + + /** ++ * List of columns that can be updated in a subscribed table ++ * without creating a new change log entry ++ * ++ * @var array ++ */ ++ private $ignoredUpdateColumns = []; ++ ++ /** + * @param ResourceConnection $resource +- * @param \Magento\Framework\DB\Ddl\TriggerFactory $triggerFactory +- * @param \Magento\Framework\Mview\View\CollectionInterface $viewCollection +- * @param \Magento\Framework\Mview\ViewInterface $view ++ * @param TriggerFactory $triggerFactory ++ * @param CollectionInterface $viewCollection ++ * @param ViewInterface $view + * @param string $tableName + * @param string $columnName ++ * @param array $ignoredUpdateColumns ++ * @throws \DomainException + */ + public function __construct( + ResourceConnection $resource, +- \Magento\Framework\DB\Ddl\TriggerFactory $triggerFactory, +- \Magento\Framework\Mview\View\CollectionInterface $viewCollection, +- \Magento\Framework\Mview\ViewInterface $view, ++ TriggerFactory $triggerFactory, ++ CollectionInterface $viewCollection, ++ ViewInterface $view, + $tableName, +- $columnName ++ $columnName, ++ array $ignoredUpdateColumns = [] + ) { + $this->connection = $resource->getConnection(); + $this->triggerFactory = $triggerFactory; +@@ -80,12 +93,14 @@ class Subscription implements SubscriptionInterface + $this->tableName = $tableName; + $this->columnName = $columnName; + $this->resource = $resource; ++ $this->ignoredUpdateColumns = $ignoredUpdateColumns; + } + + /** +- * Create subsciption ++ * Create subscription + * +- * @return \Magento\Framework\Mview\View\SubscriptionInterface ++ * @return SubscriptionInterface ++ * @throws \InvalidArgumentException + */ + public function create() + { +@@ -102,7 +117,7 @@ class Subscription implements SubscriptionInterface + + // Add statements for linked views + foreach ($this->getLinkedViews() as $view) { +- /** @var \Magento\Framework\Mview\ViewInterface $view */ ++ /** @var ViewInterface $view */ + $trigger->addStatement($this->buildStatement($event, $view->getChangelog())); + } + +@@ -116,7 +131,8 @@ class Subscription implements SubscriptionInterface + /** + * Remove subscription + * +- * @return \Magento\Framework\Mview\View\SubscriptionInterface ++ * @return SubscriptionInterface ++ * @throws \InvalidArgumentException + */ + public function remove() + { +@@ -131,7 +147,7 @@ class Subscription implements SubscriptionInterface + + // Add statements for linked views + foreach ($this->getLinkedViews() as $view) { +- /** @var \Magento\Framework\Mview\ViewInterface $view */ ++ /** @var ViewInterface $view */ + $trigger->addStatement($this->buildStatement($event, $view->getChangelog())); + } + +@@ -154,10 +170,10 @@ class Subscription implements SubscriptionInterface + protected function getLinkedViews() + { + if (!$this->linkedViews) { +- $viewList = $this->viewCollection->getViewsByStateMode(\Magento\Framework\Mview\View\StateInterface::MODE_ENABLED); ++ $viewList = $this->viewCollection->getViewsByStateMode(StateInterface::MODE_ENABLED); + + foreach ($viewList as $view) { +- /** @var \Magento\Framework\Mview\ViewInterface $view */ ++ /** @var ViewInterface $view */ + // Skip the current view + if ($view->getId() == $this->getView()->getId()) { + continue; +@@ -175,35 +191,58 @@ class Subscription implements SubscriptionInterface + } + + /** +- * Build trigger statement for INSER, UPDATE, DELETE events ++ * Build trigger statement for INSERT, UPDATE, DELETE events + * + * @param string $event +- * @param \Magento\Framework\Mview\View\ChangelogInterface $changelog ++ * @param ChangelogInterface $changelog + * @return string + */ + protected function buildStatement($event, $changelog) + { + switch ($event) { + case Trigger::EVENT_INSERT: ++ $trigger = 'INSERT IGNORE INTO %s (%s) VALUES (NEW.%s);'; ++ break; ++ + case Trigger::EVENT_UPDATE: +- return sprintf( +- "INSERT IGNORE INTO %s (%s) VALUES (NEW.%s);", +- $this->connection->quoteIdentifier($this->resource->getTableName($changelog->getName())), +- $this->connection->quoteIdentifier($changelog->getColumnName()), +- $this->connection->quoteIdentifier($this->getColumnName()) +- ); ++ $trigger = 'INSERT IGNORE INTO %s (%s) VALUES (NEW.%s);'; ++ ++ if ($this->connection->isTableExists($this->getTableName()) ++ && $describe = $this->connection->describeTable($this->getTableName()) ++ ) { ++ $columnNames = array_column($describe, 'COLUMN_NAME'); ++ $columnNames = array_diff($columnNames, $this->ignoredUpdateColumns); ++ if ($columnNames) { ++ $columns = []; ++ foreach ($columnNames as $columnName) { ++ $columns[] = sprintf( ++ 'NEW.%1$s != OLD.%1$s', ++ $this->connection->quoteIdentifier($columnName) ++ ); ++ } ++ $trigger = sprintf( ++ "IF (%s) THEN %s END IF;", ++ implode(' OR ', $columns), ++ $trigger ++ ); ++ } ++ } ++ break; + + case Trigger::EVENT_DELETE: +- return sprintf( +- "INSERT IGNORE INTO %s (%s) VALUES (OLD.%s);", +- $this->connection->quoteIdentifier($this->resource->getTableName($changelog->getName())), +- $this->connection->quoteIdentifier($changelog->getColumnName()), +- $this->connection->quoteIdentifier($this->getColumnName()) +- ); ++ $trigger = 'INSERT IGNORE INTO %s (%s) VALUES (OLD.%s);'; ++ break; + + default: + return ''; + } ++ ++ return sprintf( ++ $trigger, ++ $this->connection->quoteIdentifier($this->resource->getTableName($changelog->getName())), ++ $this->connection->quoteIdentifier($changelog->getColumnName()), ++ $this->connection->quoteIdentifier($this->getColumnName()) ++ ); + } + + /** +@@ -225,7 +264,7 @@ class Subscription implements SubscriptionInterface + /** + * Retrieve View related to subscription + * +- * @return \Magento\Framework\Mview\ViewInterface ++ * @return ViewInterface + * @codeCoverageIgnore + */ + public function getView() +diff -Nuar a/vendor/magento/framework/Mview/etc/mview.xsd b/vendor/magento/framework/Mview/etc/mview.xsd +index 1dad5b3f415..b7d6bbdde68 100644 +--- a/vendor/magento/framework/Mview/etc/mview.xsd ++++ b/vendor/magento/framework/Mview/etc/mview.xsd +@@ -106,7 +106,7 @@ + + + +- Subscription model must be a valid PHP class or interface name. ++ DEPRECATED. Subscription model must be a valid PHP class or interface name. + + + +commit a85bf456be38fa943e4144309cd330e199d1e4b6 +Author: Viktor Paladiichuk +Date: Tue Nov 28 15:30:48 2017 +0200 + + MAGETWO-84444: Mview does not work with Staging + +diff -Nuar a/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php b/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php +index b3e920fb87..f9b1d1a32a 100644 +--- a/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php ++++ b/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php +@@ -6,22 +6,16 @@ + namespace Magento\CatalogStaging\Model\Mview\View\Category\Attribute; + + use Magento\Catalog\Api\Data\CategoryInterface; +-use Magento\Framework\DB\Ddl\Trigger; + use Magento\Framework\App\ResourceConnection; + use Magento\Framework\EntityManager\MetadataPool; + + /** +- * Class Subscription ++ * Class Subscription implements statement building for staged category entity attribute subscription + * @package Magento\CatalogStaging\Model\Mview\View\Category\Attribute + */ +-class Subscription extends \Magento\Framework\Mview\View\Subscription ++class Subscription extends \Magento\CatalogStaging\Model\Mview\View\Attribute\Subscription + { + /** +- * @var \Magento\Framework\EntityManager\EntityMetadata +- */ +- protected $entityMetadata; +- +- /** + * @param ResourceConnection $resource + * @param \Magento\Framework\DB\Ddl\TriggerFactory $triggerFactory + * @param \Magento\Framework\Mview\View\CollectionInterface $viewCollection +@@ -29,7 +23,8 @@ class Subscription extends \Magento\Framework\Mview\View\Subscription + * @param string $tableName + * @param string $columnName + * @param MetadataPool $metadataPool +- * @throws \Exception ++ * @param string|null $entityInterface ++ * @param array $ignoredUpdateColumns + */ + public function __construct( + ResourceConnection $resource, +@@ -38,50 +33,20 @@ class Subscription extends \Magento\Framework\Mview\View\Subscription + \Magento\Framework\Mview\ViewInterface $view, + $tableName, + $columnName, +- MetadataPool $metadataPool ++ MetadataPool $metadataPool, ++ $entityInterface = CategoryInterface::class, ++ $ignoredUpdateColumns = [] + ) { +- parent::__construct($resource, $triggerFactory, $viewCollection, $view, $tableName, $columnName); +- $this->entityMetadata = $metadataPool->getMetadata(CategoryInterface::class); +- } +- +- /** +- * Build trigger statement for INSERT, UPDATE, DELETE events +- * +- * @param string $event +- * @param \Magento\Framework\Mview\View\ChangelogInterface $changelog +- * @return string +- */ +- protected function buildStatement($event, $changelog) +- { +- $triggerBody = null; +- switch ($event) { +- case Trigger::EVENT_INSERT: +- case Trigger::EVENT_UPDATE: +- $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = NEW.%5\$s;"; +- break; +- case Trigger::EVENT_DELETE: +- $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = OLD.%5\$s;"; +- break; +- default: +- break; +- } +- $params = [ +- $this->connection->quoteIdentifier( +- $this->resource->getTableName($changelog->getName()) +- ), +- $this->connection->quoteIdentifier( +- $changelog->getColumnName() +- ), +- $this->connection->quoteIdentifier( +- $this->entityMetadata->getIdentifierField() +- ), +- $this->connection->quoteIdentifier( +- $this->resource->getTableName($this->entityMetadata->getEntityTable()) +- ), +- $this->connection->quoteIdentifier( +- $this->entityMetadata->getLinkField() +- ) +- ]; +- return vsprintf($triggerBody, $params); ++ parent::__construct( ++ $resource, ++ $triggerFactory, ++ $viewCollection, ++ $view, ++ $tableName, ++ $columnName, ++ $metadataPool, ++ $entityInterface, ++ $ignoredUpdateColumns ++ ); + } + } +diff -Nuar a/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php b/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php +index 96c0f71164..7c841853f9 100644 +--- a/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php ++++ b/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php +@@ -11,34 +11,43 @@ class SubscriptionFactory extends FrameworkSubscriptionFactory + { + /** + * @var array ++ * @deprecated 2.2.0 + */ + private $stagingEntityTables = ['catalog_product_entity', 'catalog_category_entity']; + + /** + * @var array ++ * @deprecated 2.2.0 + */ + private $versionTables; + + /** ++ * @var string[] ++ */ ++ private $subscriptionModels = []; ++ ++ /** + * @param \Magento\Framework\ObjectManagerInterface $objectManager + * @param \Magento\CatalogStaging\Model\VersionTables $versionTables ++ * @param array $subscriptionModels + */ + public function __construct( + \Magento\Framework\ObjectManagerInterface $objectManager, +- \Magento\CatalogStaging\Model\VersionTables $versionTables ++ \Magento\CatalogStaging\Model\VersionTables $versionTables, ++ $subscriptionModels = [] + ) { + parent::__construct($objectManager); + $this->versionTables = $versionTables; ++ $this->subscriptionModels = $subscriptionModels; + } + + /** +- * @param array $data +- * @return \Magento\Framework\Mview\View\CollectionInterface ++ * {@inheritdoc} + */ + public function create(array $data = []) + { +- if ($this->isStagingTable($data)) { +- $data['columnName'] = 'row_id'; ++ if (isset($data['tableName']) && isset($this->subscriptionModels[$data['tableName']])) { ++ $data['subscriptionModel'] = $this->subscriptionModels[$data['tableName']]; + } + return parent::create($data); + } +@@ -46,6 +55,7 @@ class SubscriptionFactory extends FrameworkSubscriptionFactory + /** + * @param array $data + * @return bool ++ * @deprecated + */ + protected function isStagingTable(array $data = []) + { +diff -Nuar a/vendor/magento/module-catalog-staging/Model/VersionTables.php b/vendor/magento/module-catalog-staging/Model/VersionTables.php +index c845f98b31..242aaf2f25 100644 +--- a/vendor/magento/module-catalog-staging/Model/VersionTables.php ++++ b/vendor/magento/module-catalog-staging/Model/VersionTables.php +@@ -5,6 +5,11 @@ + */ + namespace Magento\CatalogStaging\Model; + ++/** ++ * Class VersionTables stores information about staged tables. ++ * ++ * @package Magento\CatalogStaging\Model ++ */ + class VersionTables extends \Magento\Framework\DataObject + { + /** +diff -Nuar a/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php +index d595784134..d5e78767bd 100644 +--- a/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php ++++ b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php +@@ -17,11 +17,6 @@ class SubscriptionFactoryTest extends \PHPUnit_Framework_TestCase + protected $objectManagerMock; + + /** +- * @var \PHPUnit_Framework_MockObject_MockObject +- */ +- protected $versionTablesrMock; +- +- /** + * @var \Magento\CatalogStaging\Model\Mview\View\SubscriptionFactory + */ + protected $model; +@@ -29,110 +24,45 @@ class SubscriptionFactoryTest extends \PHPUnit_Framework_TestCase + protected function setUp() + { + $objectManager = new ObjectManager($this); +- +- $this->objectManagerMock = $this->getMockBuilder('Magento\Framework\ObjectManagerInterface') +- ->disableOriginalConstructor() +- ->getMock(); +- $this->versionTablesrMock = $this->getMockBuilder('Magento\CatalogStaging\Model\VersionTables') ++ $this->objectManagerMock = $this->getMockBuilder(\Magento\Framework\ObjectManagerInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->model = $objectManager->getObject( + SubscriptionFactory::class, + [ + 'objectManager' => $this->objectManagerMock, +- 'versionTables' => $this->versionTablesrMock ++ 'subscriptionModels' => [ ++ 'catalog_product_entity_int' => 'ProductEntityIntSubscription' ++ ] + ] + ); + } +- + public function testCreate() + { + $data = ['tableName' => 'catalog_product_entity_int', 'columnName' => 'entity_id']; +- $versionTables = ['catalog_product_entity_int']; +- + $expectedData = $data; +- $expectedData['columnName'] = 'row_id'; +- +- $this->versionTablesrMock->expects($this->once()) +- ->method('getVersionTables') +- ->willReturn($versionTables); +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') ++ $expectedData['columnName'] = 'entity_id'; ++ $subscriptionMock = $this->getMockBuilder(\Magento\Framework\Mview\View\SubscriptionInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->objectManagerMock->expects($this->once()) + ->method('create') +- ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) ++ ->with('ProductEntityIntSubscription', $expectedData) + ->willReturn($subscriptionMock); +- + $result = $this->model->create($data); + $this->assertEquals($subscriptionMock, $result); + } +- + public function testCreateNoTableName() + { + $data = ['columnName' => 'entity_id']; +- +- $expectedData = $data; +- +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') +- ->disableOriginalConstructor() +- ->getMock(); +- $this->objectManagerMock->expects($this->once()) +- ->method('create') +- ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) +- ->willReturn($subscriptionMock); +- +- $result = $this->model->create($data); +- $this->assertEquals($subscriptionMock, $result); +- } +- +- /** +- * @param $stagingEntityTable +- * @dataProvider tablesDataProvider +- */ +- public function testCreateStagingEntityTables($stagingEntityTable) +- { +- $data = ['tableName' => $stagingEntityTable, 'columnName' => 'entity_id']; +- + $expectedData = $data; +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') ++ $subscriptionMock = $this->getMockBuilder(\Magento\Framework\Mview\View\SubscriptionInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->objectManagerMock->expects($this->once()) + ->method('create') + ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) + ->willReturn($subscriptionMock); +- +- $result = $this->model->create($data); +- $this->assertEquals($subscriptionMock, $result); +- } +- +- public static function tablesDataProvider() +- { +- return [ +- ['catalog_product_entity'], +- ['catalog_category_entity'] +- ]; +- } +- +- public function testCreateNoVersionTable() +- { +- $data = ['tableName' => 'not_existed_table', 'columnName' => 'entity_id']; +- $versionTables = ['catalog_product_entity_int']; +- +- $expectedData = $data; +- +- $this->versionTablesrMock->expects($this->once()) +- ->method('getVersionTables') +- ->willReturn($versionTables); +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') +- ->disableOriginalConstructor() +- ->getMock(); +- $this->objectManagerMock->expects($this->once()) +- ->method('create') +- ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) +- ->willReturn($subscriptionMock); +- + $result = $this->model->create($data); + $this->assertEquals($subscriptionMock, $result); + } +diff -Nuar a/vendor/magento/module-catalog-staging/etc/di.xml b/vendor/magento/module-catalog-staging/etc/di.xml +index aea000b42f..178b0bcf63 100644 +--- a/vendor/magento/module-catalog-staging/etc/di.xml ++++ b/vendor/magento/module-catalog-staging/etc/di.xml +@@ -56,6 +56,35 @@ + + + ++ ++ ++ Magento\Catalog\Api\Data\CategoryInterface ++ ++ ++ ++ ++ Magento\Catalog\Api\Data\ProductInterface ++ ++ ++ ++ ++ ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ ++ ++ + + + +diff -Nuar a/vendor/magento/module-catalog-staging/etc/mview.xml b/vendor/magento/module-catalog-staging/etc/mview.xml +deleted file mode 100644 +index 45bb6589b6..0000000000 +--- a/vendor/magento/module-catalog-staging/etc/mview.xml ++++ /dev/null +@@ -1,23 +0,0 @@ +- +- +- +- +- +-
+-
+-
+-
+-
+- +- +- +- +-
+- +- +- +commit 0193f89517fc864c9ab5acd4b518f03b2c796a2f +Author: Viktor Paladiichuk +Date: Tue Nov 28 15:45:17 2017 +0200 + + MAGETWO-84444: Mview does not work with Staging + +diff -Nuar a/vendor/magento/module-catalog-staging/Model/Mview/View/Attribute/Subscription.php b/vendor/magento/module-catalog-staging/Model/Mview/View/Attribute/Subscription.php +new file mode 100644 +index 0000000000..7c549538c7 +--- /dev/null ++++ b/vendor/magento/module-catalog-staging/Model/Mview/View/Attribute/Subscription.php +@@ -0,0 +1,100 @@ ++entityMetadata = $metadataPool->getMetadata($entityInterface); ++ } ++ ++ /** ++ * Build trigger statement for INSERT, UPDATE, DELETE events ++ * ++ * @param string $event ++ * @param \Magento\Framework\Mview\View\ChangelogInterface $changelog ++ * @return string ++ */ ++ protected function buildStatement($event, $changelog) ++ { ++ $triggerBody = null; ++ switch ($event) { ++ case Trigger::EVENT_INSERT: ++ case Trigger::EVENT_UPDATE: ++ $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = NEW.%5\$s;"; ++ break; ++ case Trigger::EVENT_DELETE: ++ $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = OLD.%5\$s;"; ++ break; ++ default: ++ break; ++ } ++ $params = [ ++ $this->connection->quoteIdentifier( ++ $this->resource->getTableName($changelog->getName()) ++ ), ++ $this->connection->quoteIdentifier( ++ $changelog->getColumnName() ++ ), ++ $this->connection->quoteIdentifier( ++ $this->entityMetadata->getIdentifierField() ++ ), ++ $this->connection->quoteIdentifier( ++ $this->resource->getTableName($this->entityMetadata->getEntityTable()) ++ ), ++ $this->connection->quoteIdentifier( ++ $this->entityMetadata->getLinkField() ++ ) ++ ]; ++ return vsprintf($triggerBody, $params); ++ } ++} +commit a0dc6a745002eacaaf2ef57e37a25f79f65de650 +Author: Viktor Paladiichuk +Date: Tue Nov 28 15:54:17 2017 +0200 + + MAGETWO-84444: Mview does not work with Staging + +diff -Nuar a/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/Attribute/SubscriptionTest.php b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/Attribute/SubscriptionTest.php +new file mode 100644 +index 0000000000..d396829ef8 +--- /dev/null ++++ b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/Attribute/SubscriptionTest.php +@@ -0,0 +1,302 @@ ++connectionMock = $this->getMock(Mysql::class, [], [], '', false); ++ $this->resourceMock = $this->getMock(ResourceConnection::class, [], [], '', false, false); ++ $this->connectionMock->expects($this->any()) ++ ->method('quoteIdentifier') ++ ->will($this->returnArgument(0)); ++ $this->resourceMock->expects($this->atLeastOnce()) ++ ->method('getConnection') ++ ->willReturn($this->connectionMock); ++ $this->triggerFactoryMock = $this->getMock(TriggerFactory::class, [], [], '', false, false); ++ $this->viewCollectionMock = $this->getMockForAbstractClass( ++ CollectionInterface::class, ++ [], ++ '', ++ false, ++ false, ++ true, ++ [] ++ ); ++ $this->viewMock = $this->getMockForAbstractClass(ViewInterface::class, [], '', false, false, true, []); ++ $this->resourceMock->expects($this->any()) ++ ->method('getTableName') ++ ->will($this->returnArgument(0)); ++ ++ $entityInterface = 'EntityInterface'; ++ $this->entityMetadataPoolMock = $this->getMock(MetadataPool::class, [], [], '', false); ++ ++ $this->entityMetadataMock = $this->getMock(EntityMetadataInterface::class, [], [], '', false); ++ $this->entityMetadataMock->expects($this->any()) ++ ->method('getEntityTable') ++ ->will($this->returnValue('entity_table')); ++ ++ $this->entityMetadataMock->expects($this->any()) ++ ->method('getIdentifierField') ++ ->will($this->returnValue('entity_identifier')); ++ ++ $this->entityMetadataMock->expects($this->any()) ++ ->method('getLinkField') ++ ->will($this->returnValue('entity_link_field')); ++ ++ $this->entityMetadataPoolMock->expects($this->any()) ++ ->method('getMetadata') ++ ->with($entityInterface) ++ ->will($this->returnValue($this->entityMetadataMock)); ++ ++ $this->model = new SubscriptionModel( ++ $this->resourceMock, ++ $this->triggerFactoryMock, ++ $this->viewCollectionMock, ++ $this->viewMock, ++ $this->tableName, ++ 'columnName', ++ $this->entityMetadataPoolMock, ++ $entityInterface ++ ); ++ } ++ ++ /** ++ * Prepare trigger mock ++ * ++ * @param string $triggerName ++ * @return \PHPUnit_Framework_MockObject_MockObject ++ */ ++ protected function prepareTriggerMock($triggerName) ++ { ++ $triggerMock = $this->getMockBuilder(\Magento\Framework\DB\Ddl\Trigger::class) ++ ->setMethods(['setName', 'getName', 'setTime', 'setEvent', 'setTable', 'addStatement']) ++ ->disableOriginalConstructor() ++ ->getMock(); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setName') ++ ->with($triggerName) ++ ->will($this->returnSelf()); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('getName') ++ ->will($this->returnValue('triggerName')); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setTime') ++ ->with(\Magento\Framework\DB\Ddl\Trigger::TIME_AFTER) ++ ->will($this->returnSelf()); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setEvent') ++ ->will($this->returnSelf()); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setTable') ++ ->with($this->tableName) ++ ->will($this->returnSelf()); ++ return $triggerMock; ++ } ++ ++ /** ++ * Prepare expected trigger call map ++ * ++ * @param \PHPUnit_Framework_MockObject_MockObject $triggerMock ++ * @return \PHPUnit_Framework_MockObject_MockObject ++ */ ++ protected function prepareTriggerTestCallMap(\PHPUnit_Framework_MockObject_MockObject $triggerMock) ++ { ++ $triggerMock->expects($this->at(4)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ ) ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(5)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO other_test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(11)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(12)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO other_test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(18)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = OLD.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(19)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO other_test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = OLD.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ return $triggerMock; ++ } ++ ++ /** ++ * Prepare changelog mock ++ * ++ * @param string $changelogName ++ * @return \PHPUnit_Framework_MockObject_MockObject ++ */ ++ protected function prepareChangelogMock($changelogName) ++ { ++ $changelogMock = $this->getMockForAbstractClass( ++ \Magento\Framework\Mview\View\ChangelogInterface::class, ++ [], ++ '', ++ false, ++ false, ++ true, ++ [] ++ ); ++ $changelogMock->expects($this->exactly(3)) ++ ->method('getName') ++ ->will($this->returnValue($changelogName)); ++ $changelogMock->expects($this->exactly(3)) ++ ->method('getColumnName') ++ ->will($this->returnValue('entity_id')); ++ return $changelogMock; ++ } ++ ++ public function testCreate() ++ { ++ $triggerName = 'trigger_name'; ++ $this->resourceMock->expects($this->atLeastOnce())->method('getTriggerName')->willReturn($triggerName); ++ $triggerMock = $this->prepareTriggerMock($triggerName); ++ $this->prepareTriggerTestCallMap($triggerMock); ++ $changelogMock = $this->prepareChangelogMock('test_view_cl'); ++ ++ $this->viewMock->expects($this->exactly(3)) ++ ->method('getChangelog') ++ ->will($this->returnValue($changelogMock)); ++ ++ $this->triggerFactoryMock->expects($this->exactly(3)) ++ ->method('create') ++ ->will($this->returnValue($triggerMock)); ++ ++ $otherChangelogMock = $this->prepareChangelogMock('other_test_view_cl'); ++ ++ $otherViewMock = $this->getMockForAbstractClass( ++ ViewInterface::class, ++ [], ++ '', ++ false, ++ false, ++ true, ++ [] ++ ); ++ $otherViewMock->expects($this->exactly(1)) ++ ->method('getId') ++ ->will($this->returnValue('other_id')); ++ $otherViewMock->expects($this->exactly(1)) ++ ->method('getSubscriptions') ++ ->will($this->returnValue([['name' => $this->tableName], ['name' => 'otherTableName']])); ++ $otherViewMock->expects($this->any()) ++ ->method('getChangelog') ++ ->will($this->returnValue($otherChangelogMock)); ++ ++ $this->viewMock->expects($this->exactly(3)) ++ ->method('getId') ++ ->will($this->returnValue('this_id')); ++ $this->viewMock->expects($this->never()) ++ ->method('getSubscriptions'); ++ ++ $this->viewCollectionMock->expects($this->exactly(1)) ++ ->method('getViewsByStateMode') ++ ->with(StateInterface::MODE_ENABLED) ++ ->will($this->returnValue([$this->viewMock, $otherViewMock])); ++ ++ $this->connectionMock->expects($this->exactly(3)) ++ ->method('dropTrigger') ++ ->with('triggerName') ++ ->will($this->returnValue(true)); ++ $this->connectionMock->expects($this->exactly(3)) ++ ->method('createTrigger') ++ ->with($triggerMock); ++ ++ $this->model->create(); ++ } ++} diff --git a/patches/MAGETWO-84444__fix_mview_on_staging__2.1.4.patch b/patches/MAGETWO-84444__fix_mview_on_staging__2.1.4.patch new file mode 100644 index 0000000..9f2719c --- /dev/null +++ b/patches/MAGETWO-84444__fix_mview_on_staging__2.1.4.patch @@ -0,0 +1,1172 @@ +commit 9f15e97099caa27acf41e8fe9ee16d587b605478 +Author: Viktor Paladiichuk +Date: Mon Nov 27 18:21:54 2017 +0200 + + MAGETWO-84444 + +diff -Nuar a/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php b/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php +index d8094cfa8..da9a2c724 100644 +--- a/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php ++++ b/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php +@@ -62,7 +62,7 @@ class SubscriptionTest extends \PHPUnit_Framework_TestCase + + $this->resourceMock->expects($this->any()) + ->method('getTableName') +- ->willReturn($this->tableName); ++ ->will($this->returnArgument(0)); + + $this->model = new Subscription( + $this->resourceMock, +@@ -89,11 +89,15 @@ class SubscriptionTest extends \PHPUnit_Framework_TestCase + $this->assertEquals('columnName', $this->model->getColumnName()); + } + ++ /** ++ * @SuppressWarnings(PHPMD.ExcessiveMethodLength) ++ */ + public function testCreate() + { + $triggerName = 'trigger_name'; + $this->resourceMock->expects($this->atLeastOnce())->method('getTriggerName')->willReturn($triggerName); + $triggerMock = $this->getMockBuilder('Magento\Framework\DB\Ddl\Trigger') ++ ->setMethods(['setName', 'getName', 'setTime', 'setEvent', 'setTable', 'addStatement']) + ->disableOriginalConstructor() + ->getMock(); + $triggerMock->expects($this->exactly(3)) +@@ -114,8 +118,35 @@ class SubscriptionTest extends \PHPUnit_Framework_TestCase + ->method('setTable') + ->with($this->tableName) + ->will($this->returnSelf()); +- $triggerMock->expects($this->exactly(6)) ++ ++ $triggerMock->expects($this->at(4)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(5)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO other_test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(11)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(12)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO other_test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(18)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO test_view_cl (entity_id) VALUES (OLD.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(19)) + ->method('addStatement') ++ ->with("INSERT IGNORE INTO other_test_view_cl (entity_id) VALUES (OLD.columnName);") + ->will($this->returnSelf()); + + $changelogMock = $this->getMockForAbstractClass( +diff -Nuar a/vendor/magento/framework/Mview/View/Subscription.php b/vendor/magento/framework/Mview/View/Subscription.php +index 7dd440ae9..54e4aaf53 100644 +--- a/vendor/magento/framework/Mview/View/Subscription.php ++++ b/vendor/magento/framework/Mview/View/Subscription.php +@@ -10,6 +10,8 @@ namespace Magento\Framework\Mview\View; + + use Magento\Framework\App\ResourceConnection; + use Magento\Framework\DB\Ddl\Trigger; ++use Magento\Framework\DB\Ddl\TriggerFactory; ++use Magento\Framework\Mview\ViewInterface; + + class Subscription implements SubscriptionInterface + { +@@ -21,12 +23,12 @@ class Subscription implements SubscriptionInterface + protected $connection; + + /** +- * @var \Magento\Framework\DB\Ddl\TriggerFactory ++ * @var TriggerFactory + */ + protected $triggerFactory; + + /** +- * @var \Magento\Framework\Mview\View\CollectionInterface ++ * @var CollectionInterface + */ + protected $viewCollection; + +@@ -58,20 +60,31 @@ class Subscription implements SubscriptionInterface + protected $resource; + + /** ++ * List of columns that can be updated in a subscribed table ++ * without creating a new change log entry ++ * ++ * @var array ++ */ ++ private $ignoredUpdateColumns = []; ++ ++ /** + * @param ResourceConnection $resource +- * @param \Magento\Framework\DB\Ddl\TriggerFactory $triggerFactory +- * @param \Magento\Framework\Mview\View\CollectionInterface $viewCollection +- * @param \Magento\Framework\Mview\ViewInterface $view ++ * @param TriggerFactory $triggerFactory ++ * @param CollectionInterface $viewCollection ++ * @param ViewInterface $view + * @param string $tableName + * @param string $columnName ++ * @param array $ignoredUpdateColumns ++ * @throws \DomainException + */ + public function __construct( + ResourceConnection $resource, +- \Magento\Framework\DB\Ddl\TriggerFactory $triggerFactory, +- \Magento\Framework\Mview\View\CollectionInterface $viewCollection, +- \Magento\Framework\Mview\ViewInterface $view, ++ TriggerFactory $triggerFactory, ++ CollectionInterface $viewCollection, ++ ViewInterface $view, + $tableName, +- $columnName ++ $columnName, ++ array $ignoredUpdateColumns = [] + ) { + $this->connection = $resource->getConnection(); + $this->triggerFactory = $triggerFactory; +@@ -80,12 +93,14 @@ class Subscription implements SubscriptionInterface + $this->tableName = $tableName; + $this->columnName = $columnName; + $this->resource = $resource; ++ $this->ignoredUpdateColumns = $ignoredUpdateColumns; + } + + /** +- * Create subsciption ++ * Create subscription + * +- * @return \Magento\Framework\Mview\View\SubscriptionInterface ++ * @return SubscriptionInterface ++ * @throws \InvalidArgumentException + */ + public function create() + { +@@ -102,7 +117,7 @@ class Subscription implements SubscriptionInterface + + // Add statements for linked views + foreach ($this->getLinkedViews() as $view) { +- /** @var \Magento\Framework\Mview\ViewInterface $view */ ++ /** @var ViewInterface $view */ + $trigger->addStatement($this->buildStatement($event, $view->getChangelog())); + } + +@@ -116,7 +131,8 @@ class Subscription implements SubscriptionInterface + /** + * Remove subscription + * +- * @return \Magento\Framework\Mview\View\SubscriptionInterface ++ * @return SubscriptionInterface ++ * @throws \InvalidArgumentException + */ + public function remove() + { +@@ -131,7 +147,7 @@ class Subscription implements SubscriptionInterface + + // Add statements for linked views + foreach ($this->getLinkedViews() as $view) { +- /** @var \Magento\Framework\Mview\ViewInterface $view */ ++ /** @var ViewInterface $view */ + $trigger->addStatement($this->buildStatement($event, $view->getChangelog())); + } + +@@ -154,10 +170,10 @@ class Subscription implements SubscriptionInterface + protected function getLinkedViews() + { + if (!$this->linkedViews) { +- $viewList = $this->viewCollection->getViewsByStateMode(\Magento\Framework\Mview\View\StateInterface::MODE_ENABLED); ++ $viewList = $this->viewCollection->getViewsByStateMode(StateInterface::MODE_ENABLED); + + foreach ($viewList as $view) { +- /** @var \Magento\Framework\Mview\ViewInterface $view */ ++ /** @var ViewInterface $view */ + // Skip the current view + if ($view->getId() == $this->getView()->getId()) { + continue; +@@ -175,35 +191,58 @@ class Subscription implements SubscriptionInterface + } + + /** +- * Build trigger statement for INSER, UPDATE, DELETE events ++ * Build trigger statement for INSERT, UPDATE, DELETE events + * + * @param string $event +- * @param \Magento\Framework\Mview\View\ChangelogInterface $changelog ++ * @param ChangelogInterface $changelog + * @return string + */ + protected function buildStatement($event, $changelog) + { + switch ($event) { + case Trigger::EVENT_INSERT: ++ $trigger = 'INSERT IGNORE INTO %s (%s) VALUES (NEW.%s);'; ++ break; ++ + case Trigger::EVENT_UPDATE: +- return sprintf( +- "INSERT IGNORE INTO %s (%s) VALUES (NEW.%s);", +- $this->connection->quoteIdentifier($this->resource->getTableName($changelog->getName())), +- $this->connection->quoteIdentifier($changelog->getColumnName()), +- $this->connection->quoteIdentifier($this->getColumnName()) +- ); ++ $trigger = 'INSERT IGNORE INTO %s (%s) VALUES (NEW.%s);'; ++ ++ if ($this->connection->isTableExists($this->getTableName()) ++ && $describe = $this->connection->describeTable($this->getTableName()) ++ ) { ++ $columnNames = array_column($describe, 'COLUMN_NAME'); ++ $columnNames = array_diff($columnNames, $this->ignoredUpdateColumns); ++ if ($columnNames) { ++ $columns = []; ++ foreach ($columnNames as $columnName) { ++ $columns[] = sprintf( ++ 'NEW.%1$s != OLD.%1$s', ++ $this->connection->quoteIdentifier($columnName) ++ ); ++ } ++ $trigger = sprintf( ++ "IF (%s) THEN %s END IF;", ++ implode(' OR ', $columns), ++ $trigger ++ ); ++ } ++ } ++ break; + + case Trigger::EVENT_DELETE: +- return sprintf( +- "INSERT IGNORE INTO %s (%s) VALUES (OLD.%s);", +- $this->connection->quoteIdentifier($this->resource->getTableName($changelog->getName())), +- $this->connection->quoteIdentifier($changelog->getColumnName()), +- $this->connection->quoteIdentifier($this->getColumnName()) +- ); ++ $trigger = 'INSERT IGNORE INTO %s (%s) VALUES (OLD.%s);'; ++ break; + + default: + return ''; + } ++ ++ return sprintf( ++ $trigger, ++ $this->connection->quoteIdentifier($this->resource->getTableName($changelog->getName())), ++ $this->connection->quoteIdentifier($changelog->getColumnName()), ++ $this->connection->quoteIdentifier($this->getColumnName()) ++ ); + } + + /** +@@ -225,7 +264,7 @@ class Subscription implements SubscriptionInterface + /** + * Retrieve View related to subscription + * +- * @return \Magento\Framework\Mview\ViewInterface ++ * @return ViewInterface + * @codeCoverageIgnore + */ + public function getView() +diff -Nuar a/vendor/magento/framework/Mview/etc/mview.xsd b/vendor/magento/framework/Mview/etc/mview.xsd +index d171699c3..0521691e8 100644 +--- a/vendor/magento/framework/Mview/etc/mview.xsd ++++ b/vendor/magento/framework/Mview/etc/mview.xsd +@@ -106,7 +106,7 @@ + + + +- Subscription model must be a valid PHP class or interface name. ++ DEPRECATED. Subscription model must be a valid PHP class or interface name. + + + +diff -Nuar a/vendor/magento/module-catalog-inventory/etc/mview.xml b/vendor/magento/module-catalog-inventory/etc/mview.xml +index 5737fea21..954a3349e 100644 +--- a/vendor/magento/module-catalog-inventory/etc/mview.xml ++++ b/vendor/magento/module-catalog-inventory/etc/mview.xml +@@ -5,10 +5,13 @@ + * See COPYING.txt for license details. + */ + --> +- ++ + + +
++
++
+ + + +diff -Nuar a/vendor/magento/module-catalog-staging/Model/Mview/View/Attribute/Subscription.php b/vendor/magento/module-catalog-staging/Model/Mview/View/Attribute/Subscription.php +new file mode 100644 +index 000000000..7c549538c +--- /dev/null ++++ b/vendor/magento/module-catalog-staging/Model/Mview/View/Attribute/Subscription.php +@@ -0,0 +1,100 @@ ++entityMetadata = $metadataPool->getMetadata($entityInterface); ++ } ++ ++ /** ++ * Build trigger statement for INSERT, UPDATE, DELETE events ++ * ++ * @param string $event ++ * @param \Magento\Framework\Mview\View\ChangelogInterface $changelog ++ * @return string ++ */ ++ protected function buildStatement($event, $changelog) ++ { ++ $triggerBody = null; ++ switch ($event) { ++ case Trigger::EVENT_INSERT: ++ case Trigger::EVENT_UPDATE: ++ $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = NEW.%5\$s;"; ++ break; ++ case Trigger::EVENT_DELETE: ++ $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = OLD.%5\$s;"; ++ break; ++ default: ++ break; ++ } ++ $params = [ ++ $this->connection->quoteIdentifier( ++ $this->resource->getTableName($changelog->getName()) ++ ), ++ $this->connection->quoteIdentifier( ++ $changelog->getColumnName() ++ ), ++ $this->connection->quoteIdentifier( ++ $this->entityMetadata->getIdentifierField() ++ ), ++ $this->connection->quoteIdentifier( ++ $this->resource->getTableName($this->entityMetadata->getEntityTable()) ++ ), ++ $this->connection->quoteIdentifier( ++ $this->entityMetadata->getLinkField() ++ ) ++ ]; ++ return vsprintf($triggerBody, $params); ++ } ++} +diff -Nuar a/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php b/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php +index 3ffa1a1fc..438318597 100644 +--- a/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php ++++ b/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php +@@ -6,22 +6,16 @@ + namespace Magento\CatalogStaging\Model\Mview\View\Category\Attribute; + + use Magento\Catalog\Api\Data\CategoryInterface; +-use Magento\Framework\DB\Ddl\Trigger; + use Magento\Framework\App\ResourceConnection; + use Magento\Framework\EntityManager\MetadataPool; + + /** +- * Class Subscription ++ * Class Subscription implements statement building for staged category entity attribute subscription + * @package Magento\CatalogStaging\Model\Mview\View\Category\Attribute + */ +-class Subscription extends \Magento\Framework\Mview\View\Subscription ++class Subscription extends \Magento\CatalogStaging\Model\Mview\View\Attribute\Subscription + { + /** +- * @var \Magento\Framework\EntityManager\EntityMetadata +- */ +- protected $entityMetadata; +- +- /** + * @param ResourceConnection $resource + * @param \Magento\Framework\DB\Ddl\TriggerFactory $triggerFactory + * @param \Magento\Framework\Mview\View\CollectionInterface $viewCollection +@@ -29,7 +23,8 @@ class Subscription extends \Magento\Framework\Mview\View\Subscription + * @param string $tableName + * @param string $columnName + * @param MetadataPool $metadataPool +- * @throws \Exception ++ * @param string|null $entityInterface ++ * @param array $ignoredUpdateColumns + */ + public function __construct( + ResourceConnection $resource, +@@ -38,50 +33,20 @@ class Subscription extends \Magento\Framework\Mview\View\Subscription + \Magento\Framework\Mview\ViewInterface $view, + $tableName, + $columnName, +- MetadataPool $metadataPool ++ MetadataPool $metadataPool, ++ $entityInterface = CategoryInterface::class, ++ $ignoredUpdateColumns = [] + ) { +- parent::__construct($resource, $triggerFactory, $viewCollection, $view, $tableName, $columnName); +- $this->entityMetadata = $metadataPool->getMetadata(CategoryInterface::class); +- } +- +- /** +- * Build trigger statement for INSERT, UPDATE, DELETE events +- * +- * @param string $event +- * @param \Magento\Framework\Mview\View\ChangelogInterface $changelog +- * @return string +- */ +- protected function buildStatement($event, $changelog) +- { +- $triggerBody = null; +- switch ($event) { +- case Trigger::EVENT_INSERT: +- case Trigger::EVENT_UPDATE: +- $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = NEW.%5\$s;"; +- break; +- case Trigger::EVENT_DELETE: +- $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = OLD.%5\$s;"; +- break; +- default: +- break; +- } +- $params = [ +- $this->connection->quoteIdentifier( +- $this->resource->getTableName($changelog->getName()) +- ), +- $this->connection->quoteIdentifier( +- $changelog->getColumnName() +- ), +- $this->connection->quoteIdentifier( +- $this->entityMetadata->getIdentifierField() +- ), +- $this->connection->quoteIdentifier( +- $this->resource->getTableName($this->entityMetadata->getEntityTable()) +- ), +- $this->connection->quoteIdentifier( +- $this->entityMetadata->getLinkField() +- ) +- ]; +- return vsprintf($triggerBody, $params); ++ parent::__construct( ++ $resource, ++ $triggerFactory, ++ $viewCollection, ++ $view, ++ $tableName, ++ $columnName, ++ $metadataPool, ++ $entityInterface, ++ $ignoredUpdateColumns ++ ); + } + } +diff -Nuar a/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php b/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php +index 6f03681f7..052aa3702 100644 +--- a/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php ++++ b/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php +@@ -11,34 +11,43 @@ class SubscriptionFactory extends FrameworkSubscriptionFactory + { + /** + * @var array ++ * @deprecated 2.2.0 + */ + private $stagingEntityTables = ['catalog_product_entity', 'catalog_category_entity']; + + /** + * @var array ++ * @deprecated 2.2.0 + */ + private $versionTables; + + /** ++ * @var string[] ++ */ ++ private $subscriptionModels = []; ++ ++ /** + * @param \Magento\Framework\ObjectManagerInterface $objectManager + * @param \Magento\CatalogStaging\Model\VersionTables $versionTables ++ * @param array $subscriptionModels + */ + public function __construct( + \Magento\Framework\ObjectManagerInterface $objectManager, +- \Magento\CatalogStaging\Model\VersionTables $versionTables ++ \Magento\CatalogStaging\Model\VersionTables $versionTables, ++ $subscriptionModels = [] + ) { + parent::__construct($objectManager); + $this->versionTables = $versionTables; ++ $this->subscriptionModels = $subscriptionModels; + } + + /** +- * @param array $data +- * @return \Magento\Framework\Mview\View\CollectionInterface ++ * {@inheritdoc} + */ + public function create(array $data = []) + { +- if ($this->isStagingTable($data)) { +- $data['columnName'] = 'row_id'; ++ if (isset($data['tableName']) && isset($this->subscriptionModels[$data['tableName']])) { ++ $data['subscriptionModel'] = $this->subscriptionModels[$data['tableName']]; + } + return parent::create($data); + } +@@ -46,6 +55,7 @@ class SubscriptionFactory extends FrameworkSubscriptionFactory + /** + * @param array $data + * @return bool ++ * @deprecated + */ + protected function isStagingTable(array $data = []) + { +diff -Nuar a/vendor/magento/module-catalog-staging/Model/VersionTables.php b/vendor/magento/module-catalog-staging/Model/VersionTables.php +index 7ebd828bd..2cc32669b 100644 +--- a/vendor/magento/module-catalog-staging/Model/VersionTables.php ++++ b/vendor/magento/module-catalog-staging/Model/VersionTables.php +@@ -5,6 +5,11 @@ + */ + namespace Magento\CatalogStaging\Model; + ++/** ++ * Class VersionTables stores information about staged tables. ++ * ++ * @package Magento\CatalogStaging\Model ++ */ + class VersionTables extends \Magento\Framework\DataObject + { + /** +diff -Nuar a/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/Attribute/SubscriptionTest.php b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/Attribute/SubscriptionTest.php +new file mode 100644 +index 000000000..d396829ef +--- /dev/null ++++ b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/Attribute/SubscriptionTest.php +@@ -0,0 +1,302 @@ ++connectionMock = $this->getMock(Mysql::class, [], [], '', false); ++ $this->resourceMock = $this->getMock(ResourceConnection::class, [], [], '', false, false); ++ $this->connectionMock->expects($this->any()) ++ ->method('quoteIdentifier') ++ ->will($this->returnArgument(0)); ++ $this->resourceMock->expects($this->atLeastOnce()) ++ ->method('getConnection') ++ ->willReturn($this->connectionMock); ++ $this->triggerFactoryMock = $this->getMock(TriggerFactory::class, [], [], '', false, false); ++ $this->viewCollectionMock = $this->getMockForAbstractClass( ++ CollectionInterface::class, ++ [], ++ '', ++ false, ++ false, ++ true, ++ [] ++ ); ++ $this->viewMock = $this->getMockForAbstractClass(ViewInterface::class, [], '', false, false, true, []); ++ $this->resourceMock->expects($this->any()) ++ ->method('getTableName') ++ ->will($this->returnArgument(0)); ++ ++ $entityInterface = 'EntityInterface'; ++ $this->entityMetadataPoolMock = $this->getMock(MetadataPool::class, [], [], '', false); ++ ++ $this->entityMetadataMock = $this->getMock(EntityMetadataInterface::class, [], [], '', false); ++ $this->entityMetadataMock->expects($this->any()) ++ ->method('getEntityTable') ++ ->will($this->returnValue('entity_table')); ++ ++ $this->entityMetadataMock->expects($this->any()) ++ ->method('getIdentifierField') ++ ->will($this->returnValue('entity_identifier')); ++ ++ $this->entityMetadataMock->expects($this->any()) ++ ->method('getLinkField') ++ ->will($this->returnValue('entity_link_field')); ++ ++ $this->entityMetadataPoolMock->expects($this->any()) ++ ->method('getMetadata') ++ ->with($entityInterface) ++ ->will($this->returnValue($this->entityMetadataMock)); ++ ++ $this->model = new SubscriptionModel( ++ $this->resourceMock, ++ $this->triggerFactoryMock, ++ $this->viewCollectionMock, ++ $this->viewMock, ++ $this->tableName, ++ 'columnName', ++ $this->entityMetadataPoolMock, ++ $entityInterface ++ ); ++ } ++ ++ /** ++ * Prepare trigger mock ++ * ++ * @param string $triggerName ++ * @return \PHPUnit_Framework_MockObject_MockObject ++ */ ++ protected function prepareTriggerMock($triggerName) ++ { ++ $triggerMock = $this->getMockBuilder(\Magento\Framework\DB\Ddl\Trigger::class) ++ ->setMethods(['setName', 'getName', 'setTime', 'setEvent', 'setTable', 'addStatement']) ++ ->disableOriginalConstructor() ++ ->getMock(); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setName') ++ ->with($triggerName) ++ ->will($this->returnSelf()); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('getName') ++ ->will($this->returnValue('triggerName')); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setTime') ++ ->with(\Magento\Framework\DB\Ddl\Trigger::TIME_AFTER) ++ ->will($this->returnSelf()); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setEvent') ++ ->will($this->returnSelf()); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setTable') ++ ->with($this->tableName) ++ ->will($this->returnSelf()); ++ return $triggerMock; ++ } ++ ++ /** ++ * Prepare expected trigger call map ++ * ++ * @param \PHPUnit_Framework_MockObject_MockObject $triggerMock ++ * @return \PHPUnit_Framework_MockObject_MockObject ++ */ ++ protected function prepareTriggerTestCallMap(\PHPUnit_Framework_MockObject_MockObject $triggerMock) ++ { ++ $triggerMock->expects($this->at(4)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ ) ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(5)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO other_test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(11)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(12)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO other_test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(18)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = OLD.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(19)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO other_test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = OLD.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ return $triggerMock; ++ } ++ ++ /** ++ * Prepare changelog mock ++ * ++ * @param string $changelogName ++ * @return \PHPUnit_Framework_MockObject_MockObject ++ */ ++ protected function prepareChangelogMock($changelogName) ++ { ++ $changelogMock = $this->getMockForAbstractClass( ++ \Magento\Framework\Mview\View\ChangelogInterface::class, ++ [], ++ '', ++ false, ++ false, ++ true, ++ [] ++ ); ++ $changelogMock->expects($this->exactly(3)) ++ ->method('getName') ++ ->will($this->returnValue($changelogName)); ++ $changelogMock->expects($this->exactly(3)) ++ ->method('getColumnName') ++ ->will($this->returnValue('entity_id')); ++ return $changelogMock; ++ } ++ ++ public function testCreate() ++ { ++ $triggerName = 'trigger_name'; ++ $this->resourceMock->expects($this->atLeastOnce())->method('getTriggerName')->willReturn($triggerName); ++ $triggerMock = $this->prepareTriggerMock($triggerName); ++ $this->prepareTriggerTestCallMap($triggerMock); ++ $changelogMock = $this->prepareChangelogMock('test_view_cl'); ++ ++ $this->viewMock->expects($this->exactly(3)) ++ ->method('getChangelog') ++ ->will($this->returnValue($changelogMock)); ++ ++ $this->triggerFactoryMock->expects($this->exactly(3)) ++ ->method('create') ++ ->will($this->returnValue($triggerMock)); ++ ++ $otherChangelogMock = $this->prepareChangelogMock('other_test_view_cl'); ++ ++ $otherViewMock = $this->getMockForAbstractClass( ++ ViewInterface::class, ++ [], ++ '', ++ false, ++ false, ++ true, ++ [] ++ ); ++ $otherViewMock->expects($this->exactly(1)) ++ ->method('getId') ++ ->will($this->returnValue('other_id')); ++ $otherViewMock->expects($this->exactly(1)) ++ ->method('getSubscriptions') ++ ->will($this->returnValue([['name' => $this->tableName], ['name' => 'otherTableName']])); ++ $otherViewMock->expects($this->any()) ++ ->method('getChangelog') ++ ->will($this->returnValue($otherChangelogMock)); ++ ++ $this->viewMock->expects($this->exactly(3)) ++ ->method('getId') ++ ->will($this->returnValue('this_id')); ++ $this->viewMock->expects($this->never()) ++ ->method('getSubscriptions'); ++ ++ $this->viewCollectionMock->expects($this->exactly(1)) ++ ->method('getViewsByStateMode') ++ ->with(StateInterface::MODE_ENABLED) ++ ->will($this->returnValue([$this->viewMock, $otherViewMock])); ++ ++ $this->connectionMock->expects($this->exactly(3)) ++ ->method('dropTrigger') ++ ->with('triggerName') ++ ->will($this->returnValue(true)); ++ $this->connectionMock->expects($this->exactly(3)) ++ ->method('createTrigger') ++ ->with($triggerMock); ++ ++ $this->model->create(); ++ } ++} +diff -Nuar a/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php +index a0abb4230..4a0601569 100644 +--- a/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php ++++ b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php +@@ -17,11 +17,6 @@ class SubscriptionFactoryTest extends \PHPUnit_Framework_TestCase + protected $objectManagerMock; + + /** +- * @var \PHPUnit_Framework_MockObject_MockObject +- */ +- protected $versionTablesrMock; +- +- /** + * @var \Magento\CatalogStaging\Model\Mview\View\SubscriptionFactory + */ + protected $model; +@@ -29,110 +24,45 @@ class SubscriptionFactoryTest extends \PHPUnit_Framework_TestCase + protected function setUp() + { + $objectManager = new ObjectManager($this); +- +- $this->objectManagerMock = $this->getMockBuilder('Magento\Framework\ObjectManagerInterface') +- ->disableOriginalConstructor() +- ->getMock(); +- $this->versionTablesrMock = $this->getMockBuilder('Magento\CatalogStaging\Model\VersionTables') ++ $this->objectManagerMock = $this->getMockBuilder(\Magento\Framework\ObjectManagerInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->model = $objectManager->getObject( + SubscriptionFactory::class, + [ + 'objectManager' => $this->objectManagerMock, +- 'versionTables' => $this->versionTablesrMock ++ 'subscriptionModels' => [ ++ 'catalog_product_entity_int' => 'ProductEntityIntSubscription' ++ ] + ] + ); + } +- + public function testCreate() + { + $data = ['tableName' => 'catalog_product_entity_int', 'columnName' => 'entity_id']; +- $versionTables = ['catalog_product_entity_int']; +- + $expectedData = $data; +- $expectedData['columnName'] = 'row_id'; +- +- $this->versionTablesrMock->expects($this->once()) +- ->method('getVersionTables') +- ->willReturn($versionTables); +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') ++ $expectedData['columnName'] = 'entity_id'; ++ $subscriptionMock = $this->getMockBuilder(\Magento\Framework\Mview\View\SubscriptionInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->objectManagerMock->expects($this->once()) + ->method('create') +- ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) ++ ->with('ProductEntityIntSubscription', $expectedData) + ->willReturn($subscriptionMock); +- + $result = $this->model->create($data); + $this->assertEquals($subscriptionMock, $result); + } +- + public function testCreateNoTableName() + { + $data = ['columnName' => 'entity_id']; +- +- $expectedData = $data; +- +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') +- ->disableOriginalConstructor() +- ->getMock(); +- $this->objectManagerMock->expects($this->once()) +- ->method('create') +- ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) +- ->willReturn($subscriptionMock); +- +- $result = $this->model->create($data); +- $this->assertEquals($subscriptionMock, $result); +- } +- +- /** +- * @param $stagingEntityTable +- * @dataProvider tablesDataProvider +- */ +- public function testCreateStagingEntityTables($stagingEntityTable) +- { +- $data = ['tableName' => $stagingEntityTable, 'columnName' => 'entity_id']; +- + $expectedData = $data; +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') ++ $subscriptionMock = $this->getMockBuilder(\Magento\Framework\Mview\View\SubscriptionInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->objectManagerMock->expects($this->once()) + ->method('create') + ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) + ->willReturn($subscriptionMock); +- +- $result = $this->model->create($data); +- $this->assertEquals($subscriptionMock, $result); +- } +- +- public static function tablesDataProvider() +- { +- return [ +- ['catalog_product_entity'], +- ['catalog_category_entity'] +- ]; +- } +- +- public function testCreateNoVersionTable() +- { +- $data = ['tableName' => 'not_existed_table', 'columnName' => 'entity_id']; +- $versionTables = ['catalog_product_entity_int']; +- +- $expectedData = $data; +- +- $this->versionTablesrMock->expects($this->once()) +- ->method('getVersionTables') +- ->willReturn($versionTables); +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') +- ->disableOriginalConstructor() +- ->getMock(); +- $this->objectManagerMock->expects($this->once()) +- ->method('create') +- ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) +- ->willReturn($subscriptionMock); +- + $result = $this->model->create($data); + $this->assertEquals($subscriptionMock, $result); + } +diff -Nuar a/vendor/magento/module-catalog-staging/etc/di.xml b/vendor/magento/module-catalog-staging/etc/di.xml +index 79f95ec08..e28ac70e7 100644 +--- a/vendor/magento/module-catalog-staging/etc/di.xml ++++ b/vendor/magento/module-catalog-staging/etc/di.xml +@@ -56,6 +56,35 @@ + + + ++ ++ ++ Magento\Catalog\Api\Data\CategoryInterface ++ ++ ++ ++ ++ Magento\Catalog\Api\Data\ProductInterface ++ ++ ++ ++ ++ ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ ++ ++ + + + +diff -Nuar a/vendor/magento/module-catalog-staging/etc/mview.xml b/vendor/magento/module-catalog-staging/etc/mview.xml +deleted file mode 100644 +index 488972e05..000000000 +--- a/vendor/magento/module-catalog-staging/etc/mview.xml ++++ /dev/null +@@ -1,23 +0,0 @@ +- +- +- +- +- +-
+-
+-
+-
+-
+- +- +- +- +-
+- +- +- +diff -Nuar a/vendor/magento/module-indexer/Setup/RecurringData.php b/vendor/magento/module-indexer/Setup/RecurringData.php +new file mode 100644 +index 000000000..38ea0e5b7 +--- /dev/null ++++ b/vendor/magento/module-indexer/Setup/RecurringData.php +@@ -0,0 +1,56 @@ ++indexerFactory = $indexerFactory; ++ $this->configInterface = $configInterface; ++ } ++ ++ /** ++ * {@inheritdoc} ++ */ ++ public function install(ModuleDataSetupInterface $setup, ModuleContextInterface $context) ++ { ++ foreach (array_keys($this->configInterface->getIndexers()) as $indexerId) { ++ $indexer = $this->indexerFactory->create()->load($indexerId); ++ if ($indexer->isScheduled()) { ++ $indexer->getView()->unsubscribe()->subscribe(); ++ } ++ } ++ } ++} diff --git a/patches/MAGETWO-84444__fix_mview_on_staging__2.1.5.patch b/patches/MAGETWO-84444__fix_mview_on_staging__2.1.5.patch new file mode 100644 index 0000000..c044017 --- /dev/null +++ b/patches/MAGETWO-84444__fix_mview_on_staging__2.1.5.patch @@ -0,0 +1,1190 @@ +commit e9aa4e18cfb76b37ce00f6f45d507d22e1e89550 +Author: Viktor Paladiichuk +Date: Tue Nov 28 15:29:54 2017 +0200 + + MAGETWO-84444: Mview does not work with Staging + +diff -Nuar a/vendor/magento/module-catalog-inventory/etc/mview.xml b/vendor/magento/module-catalog-inventory/etc/mview.xml +index 58a051a3d0e..3dd8419d7e3 100644 +--- a/vendor/magento/module-catalog-inventory/etc/mview.xml ++++ b/vendor/magento/module-catalog-inventory/etc/mview.xml +@@ -5,10 +5,13 @@ + * See COPYING.txt for license details. + */ + --> +- ++ + + +
++
++
+ + + +diff -Nuar a/vendor/magento/module-indexer/Setup/RecurringData.php b/vendor/magento/module-indexer/Setup/RecurringData.php +new file mode 100644 +index 00000000000..38ea0e5b79e +--- /dev/null ++++ b/vendor/magento/module-indexer/Setup/RecurringData.php +@@ -0,0 +1,56 @@ ++indexerFactory = $indexerFactory; ++ $this->configInterface = $configInterface; ++ } ++ ++ /** ++ * {@inheritdoc} ++ */ ++ public function install(ModuleDataSetupInterface $setup, ModuleContextInterface $context) ++ { ++ foreach (array_keys($this->configInterface->getIndexers()) as $indexerId) { ++ $indexer = $this->indexerFactory->create()->load($indexerId); ++ if ($indexer->isScheduled()) { ++ $indexer->getView()->unsubscribe()->subscribe(); ++ } ++ } ++ } ++} +diff -Nuar a/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php b/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php +index aae82938c83..545689a4391 100644 +--- a/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php ++++ b/vendor/magento/framework/Mview/Test/Unit/View/SubscriptionTest.php +@@ -62,7 +62,7 @@ class SubscriptionTest extends \PHPUnit_Framework_TestCase + + $this->resourceMock->expects($this->any()) + ->method('getTableName') +- ->willReturn($this->tableName); ++ ->will($this->returnArgument(0)); + + $this->model = new Subscription( + $this->resourceMock, +@@ -89,11 +89,15 @@ class SubscriptionTest extends \PHPUnit_Framework_TestCase + $this->assertEquals('columnName', $this->model->getColumnName()); + } + ++ /** ++ * @SuppressWarnings(PHPMD.ExcessiveMethodLength) ++ */ + public function testCreate() + { + $triggerName = 'trigger_name'; + $this->resourceMock->expects($this->atLeastOnce())->method('getTriggerName')->willReturn($triggerName); + $triggerMock = $this->getMockBuilder('Magento\Framework\DB\Ddl\Trigger') ++ ->setMethods(['setName', 'getName', 'setTime', 'setEvent', 'setTable', 'addStatement']) + ->disableOriginalConstructor() + ->getMock(); + $triggerMock->expects($this->exactly(3)) +@@ -114,8 +118,35 @@ class SubscriptionTest extends \PHPUnit_Framework_TestCase + ->method('setTable') + ->with($this->tableName) + ->will($this->returnSelf()); +- $triggerMock->expects($this->exactly(6)) ++ ++ $triggerMock->expects($this->at(4)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(5)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO other_test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(11)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(12)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO other_test_view_cl (entity_id) VALUES (NEW.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(18)) ++ ->method('addStatement') ++ ->with("INSERT IGNORE INTO test_view_cl (entity_id) VALUES (OLD.columnName);") ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(19)) + ->method('addStatement') ++ ->with("INSERT IGNORE INTO other_test_view_cl (entity_id) VALUES (OLD.columnName);") + ->will($this->returnSelf()); + + $changelogMock = $this->getMockForAbstractClass( +diff -Nuar a/vendor/magento/framework/Mview/View/Subscription.php b/vendor/magento/framework/Mview/View/Subscription.php +index c3da91c8331..3c4bd1dce2d 100644 +--- a/vendor/magento/framework/Mview/View/Subscription.php ++++ b/vendor/magento/framework/Mview/View/Subscription.php +@@ -10,6 +10,8 @@ namespace Magento\Framework\Mview\View; + + use Magento\Framework\App\ResourceConnection; + use Magento\Framework\DB\Ddl\Trigger; ++use Magento\Framework\DB\Ddl\TriggerFactory; ++use Magento\Framework\Mview\ViewInterface; + + class Subscription implements SubscriptionInterface + { +@@ -21,12 +23,12 @@ class Subscription implements SubscriptionInterface + protected $connection; + + /** +- * @var \Magento\Framework\DB\Ddl\TriggerFactory ++ * @var TriggerFactory + */ + protected $triggerFactory; + + /** +- * @var \Magento\Framework\Mview\View\CollectionInterface ++ * @var CollectionInterface + */ + protected $viewCollection; + +@@ -58,20 +60,31 @@ class Subscription implements SubscriptionInterface + protected $resource; + + /** ++ * List of columns that can be updated in a subscribed table ++ * without creating a new change log entry ++ * ++ * @var array ++ */ ++ private $ignoredUpdateColumns = []; ++ ++ /** + * @param ResourceConnection $resource +- * @param \Magento\Framework\DB\Ddl\TriggerFactory $triggerFactory +- * @param \Magento\Framework\Mview\View\CollectionInterface $viewCollection +- * @param \Magento\Framework\Mview\ViewInterface $view ++ * @param TriggerFactory $triggerFactory ++ * @param CollectionInterface $viewCollection ++ * @param ViewInterface $view + * @param string $tableName + * @param string $columnName ++ * @param array $ignoredUpdateColumns ++ * @throws \DomainException + */ + public function __construct( + ResourceConnection $resource, +- \Magento\Framework\DB\Ddl\TriggerFactory $triggerFactory, +- \Magento\Framework\Mview\View\CollectionInterface $viewCollection, +- \Magento\Framework\Mview\ViewInterface $view, ++ TriggerFactory $triggerFactory, ++ CollectionInterface $viewCollection, ++ ViewInterface $view, + $tableName, +- $columnName ++ $columnName, ++ array $ignoredUpdateColumns = [] + ) { + $this->connection = $resource->getConnection(); + $this->triggerFactory = $triggerFactory; +@@ -80,12 +93,14 @@ class Subscription implements SubscriptionInterface + $this->tableName = $tableName; + $this->columnName = $columnName; + $this->resource = $resource; ++ $this->ignoredUpdateColumns = $ignoredUpdateColumns; + } + + /** +- * Create subsciption ++ * Create subscription + * +- * @return \Magento\Framework\Mview\View\SubscriptionInterface ++ * @return SubscriptionInterface ++ * @throws \InvalidArgumentException + */ + public function create() + { +@@ -102,7 +117,7 @@ class Subscription implements SubscriptionInterface + + // Add statements for linked views + foreach ($this->getLinkedViews() as $view) { +- /** @var \Magento\Framework\Mview\ViewInterface $view */ ++ /** @var ViewInterface $view */ + $trigger->addStatement($this->buildStatement($event, $view->getChangelog())); + } + +@@ -116,7 +131,8 @@ class Subscription implements SubscriptionInterface + /** + * Remove subscription + * +- * @return \Magento\Framework\Mview\View\SubscriptionInterface ++ * @return SubscriptionInterface ++ * @throws \InvalidArgumentException + */ + public function remove() + { +@@ -131,7 +147,7 @@ class Subscription implements SubscriptionInterface + + // Add statements for linked views + foreach ($this->getLinkedViews() as $view) { +- /** @var \Magento\Framework\Mview\ViewInterface $view */ ++ /** @var ViewInterface $view */ + $trigger->addStatement($this->buildStatement($event, $view->getChangelog())); + } + +@@ -154,10 +170,10 @@ class Subscription implements SubscriptionInterface + protected function getLinkedViews() + { + if (!$this->linkedViews) { +- $viewList = $this->viewCollection->getViewsByStateMode(\Magento\Framework\Mview\View\StateInterface::MODE_ENABLED); ++ $viewList = $this->viewCollection->getViewsByStateMode(StateInterface::MODE_ENABLED); + + foreach ($viewList as $view) { +- /** @var \Magento\Framework\Mview\ViewInterface $view */ ++ /** @var ViewInterface $view */ + // Skip the current view + if ($view->getId() == $this->getView()->getId()) { + continue; +@@ -175,35 +191,58 @@ class Subscription implements SubscriptionInterface + } + + /** +- * Build trigger statement for INSER, UPDATE, DELETE events ++ * Build trigger statement for INSERT, UPDATE, DELETE events + * + * @param string $event +- * @param \Magento\Framework\Mview\View\ChangelogInterface $changelog ++ * @param ChangelogInterface $changelog + * @return string + */ + protected function buildStatement($event, $changelog) + { + switch ($event) { + case Trigger::EVENT_INSERT: ++ $trigger = 'INSERT IGNORE INTO %s (%s) VALUES (NEW.%s);'; ++ break; ++ + case Trigger::EVENT_UPDATE: +- return sprintf( +- "INSERT IGNORE INTO %s (%s) VALUES (NEW.%s);", +- $this->connection->quoteIdentifier($this->resource->getTableName($changelog->getName())), +- $this->connection->quoteIdentifier($changelog->getColumnName()), +- $this->connection->quoteIdentifier($this->getColumnName()) +- ); ++ $trigger = 'INSERT IGNORE INTO %s (%s) VALUES (NEW.%s);'; ++ ++ if ($this->connection->isTableExists($this->getTableName()) ++ && $describe = $this->connection->describeTable($this->getTableName()) ++ ) { ++ $columnNames = array_column($describe, 'COLUMN_NAME'); ++ $columnNames = array_diff($columnNames, $this->ignoredUpdateColumns); ++ if ($columnNames) { ++ $columns = []; ++ foreach ($columnNames as $columnName) { ++ $columns[] = sprintf( ++ 'NEW.%1$s != OLD.%1$s', ++ $this->connection->quoteIdentifier($columnName) ++ ); ++ } ++ $trigger = sprintf( ++ "IF (%s) THEN %s END IF;", ++ implode(' OR ', $columns), ++ $trigger ++ ); ++ } ++ } ++ break; + + case Trigger::EVENT_DELETE: +- return sprintf( +- "INSERT IGNORE INTO %s (%s) VALUES (OLD.%s);", +- $this->connection->quoteIdentifier($this->resource->getTableName($changelog->getName())), +- $this->connection->quoteIdentifier($changelog->getColumnName()), +- $this->connection->quoteIdentifier($this->getColumnName()) +- ); ++ $trigger = 'INSERT IGNORE INTO %s (%s) VALUES (OLD.%s);'; ++ break; + + default: + return ''; + } ++ ++ return sprintf( ++ $trigger, ++ $this->connection->quoteIdentifier($this->resource->getTableName($changelog->getName())), ++ $this->connection->quoteIdentifier($changelog->getColumnName()), ++ $this->connection->quoteIdentifier($this->getColumnName()) ++ ); + } + + /** +@@ -225,7 +264,7 @@ class Subscription implements SubscriptionInterface + /** + * Retrieve View related to subscription + * +- * @return \Magento\Framework\Mview\ViewInterface ++ * @return ViewInterface + * @codeCoverageIgnore + */ + public function getView() +diff -Nuar a/vendor/magento/framework/Mview/etc/mview.xsd b/vendor/magento/framework/Mview/etc/mview.xsd +index 1dad5b3f415..b7d6bbdde68 100644 +--- a/vendor/magento/framework/Mview/etc/mview.xsd ++++ b/vendor/magento/framework/Mview/etc/mview.xsd +@@ -106,7 +106,7 @@ + + + +- Subscription model must be a valid PHP class or interface name. ++ DEPRECATED. Subscription model must be a valid PHP class or interface name. + + + +commit a85bf456be38fa943e4144309cd330e199d1e4b6 +Author: Viktor Paladiichuk +Date: Tue Nov 28 15:30:48 2017 +0200 + + MAGETWO-84444: Mview does not work with Staging + +diff -Nuar a/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php b/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php +index b3e920fb87..f9b1d1a32a 100644 +--- a/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php ++++ b/vendor/magento/module-catalog-staging/Model/Mview/View/Category/Attribute/Subscription.php +@@ -6,22 +6,16 @@ + namespace Magento\CatalogStaging\Model\Mview\View\Category\Attribute; + + use Magento\Catalog\Api\Data\CategoryInterface; +-use Magento\Framework\DB\Ddl\Trigger; + use Magento\Framework\App\ResourceConnection; + use Magento\Framework\EntityManager\MetadataPool; + + /** +- * Class Subscription ++ * Class Subscription implements statement building for staged category entity attribute subscription + * @package Magento\CatalogStaging\Model\Mview\View\Category\Attribute + */ +-class Subscription extends \Magento\Framework\Mview\View\Subscription ++class Subscription extends \Magento\CatalogStaging\Model\Mview\View\Attribute\Subscription + { + /** +- * @var \Magento\Framework\EntityManager\EntityMetadata +- */ +- protected $entityMetadata; +- +- /** + * @param ResourceConnection $resource + * @param \Magento\Framework\DB\Ddl\TriggerFactory $triggerFactory + * @param \Magento\Framework\Mview\View\CollectionInterface $viewCollection +@@ -29,7 +23,8 @@ class Subscription extends \Magento\Framework\Mview\View\Subscription + * @param string $tableName + * @param string $columnName + * @param MetadataPool $metadataPool +- * @throws \Exception ++ * @param string|null $entityInterface ++ * @param array $ignoredUpdateColumns + */ + public function __construct( + ResourceConnection $resource, +@@ -38,50 +33,20 @@ class Subscription extends \Magento\Framework\Mview\View\Subscription + \Magento\Framework\Mview\ViewInterface $view, + $tableName, + $columnName, +- MetadataPool $metadataPool ++ MetadataPool $metadataPool, ++ $entityInterface = CategoryInterface::class, ++ $ignoredUpdateColumns = [] + ) { +- parent::__construct($resource, $triggerFactory, $viewCollection, $view, $tableName, $columnName); +- $this->entityMetadata = $metadataPool->getMetadata(CategoryInterface::class); +- } +- +- /** +- * Build trigger statement for INSERT, UPDATE, DELETE events +- * +- * @param string $event +- * @param \Magento\Framework\Mview\View\ChangelogInterface $changelog +- * @return string +- */ +- protected function buildStatement($event, $changelog) +- { +- $triggerBody = null; +- switch ($event) { +- case Trigger::EVENT_INSERT: +- case Trigger::EVENT_UPDATE: +- $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = NEW.%5\$s;"; +- break; +- case Trigger::EVENT_DELETE: +- $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = OLD.%5\$s;"; +- break; +- default: +- break; +- } +- $params = [ +- $this->connection->quoteIdentifier( +- $this->resource->getTableName($changelog->getName()) +- ), +- $this->connection->quoteIdentifier( +- $changelog->getColumnName() +- ), +- $this->connection->quoteIdentifier( +- $this->entityMetadata->getIdentifierField() +- ), +- $this->connection->quoteIdentifier( +- $this->resource->getTableName($this->entityMetadata->getEntityTable()) +- ), +- $this->connection->quoteIdentifier( +- $this->entityMetadata->getLinkField() +- ) +- ]; +- return vsprintf($triggerBody, $params); ++ parent::__construct( ++ $resource, ++ $triggerFactory, ++ $viewCollection, ++ $view, ++ $tableName, ++ $columnName, ++ $metadataPool, ++ $entityInterface, ++ $ignoredUpdateColumns ++ ); + } + } +diff -Nuar a/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php b/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php +index 96c0f71164..7c841853f9 100644 +--- a/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php ++++ b/vendor/magento/module-catalog-staging/Model/Mview/View/SubscriptionFactory.php +@@ -11,34 +11,43 @@ class SubscriptionFactory extends FrameworkSubscriptionFactory + { + /** + * @var array ++ * @deprecated 2.2.0 + */ + private $stagingEntityTables = ['catalog_product_entity', 'catalog_category_entity']; + + /** + * @var array ++ * @deprecated 2.2.0 + */ + private $versionTables; + + /** ++ * @var string[] ++ */ ++ private $subscriptionModels = []; ++ ++ /** + * @param \Magento\Framework\ObjectManagerInterface $objectManager + * @param \Magento\CatalogStaging\Model\VersionTables $versionTables ++ * @param array $subscriptionModels + */ + public function __construct( + \Magento\Framework\ObjectManagerInterface $objectManager, +- \Magento\CatalogStaging\Model\VersionTables $versionTables ++ \Magento\CatalogStaging\Model\VersionTables $versionTables, ++ $subscriptionModels = [] + ) { + parent::__construct($objectManager); + $this->versionTables = $versionTables; ++ $this->subscriptionModels = $subscriptionModels; + } + + /** +- * @param array $data +- * @return \Magento\Framework\Mview\View\CollectionInterface ++ * {@inheritdoc} + */ + public function create(array $data = []) + { +- if ($this->isStagingTable($data)) { +- $data['columnName'] = 'row_id'; ++ if (isset($data['tableName']) && isset($this->subscriptionModels[$data['tableName']])) { ++ $data['subscriptionModel'] = $this->subscriptionModels[$data['tableName']]; + } + return parent::create($data); + } +@@ -46,6 +55,7 @@ class SubscriptionFactory extends FrameworkSubscriptionFactory + /** + * @param array $data + * @return bool ++ * @deprecated + */ + protected function isStagingTable(array $data = []) + { +diff -Nuar a/vendor/magento/module-catalog-staging/Model/VersionTables.php b/vendor/magento/module-catalog-staging/Model/VersionTables.php +index c845f98b31..242aaf2f25 100644 +--- a/vendor/magento/module-catalog-staging/Model/VersionTables.php ++++ b/vendor/magento/module-catalog-staging/Model/VersionTables.php +@@ -5,6 +5,11 @@ + */ + namespace Magento\CatalogStaging\Model; + ++/** ++ * Class VersionTables stores information about staged tables. ++ * ++ * @package Magento\CatalogStaging\Model ++ */ + class VersionTables extends \Magento\Framework\DataObject + { + /** +diff -Nuar a/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php +index d595784134..d5e78767bd 100644 +--- a/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php ++++ b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/SubscriptionFactoryTest.php +@@ -17,11 +17,6 @@ class SubscriptionFactoryTest extends \PHPUnit_Framework_TestCase + protected $objectManagerMock; + + /** +- * @var \PHPUnit_Framework_MockObject_MockObject +- */ +- protected $versionTablesrMock; +- +- /** + * @var \Magento\CatalogStaging\Model\Mview\View\SubscriptionFactory + */ + protected $model; +@@ -29,110 +24,45 @@ class SubscriptionFactoryTest extends \PHPUnit_Framework_TestCase + protected function setUp() + { + $objectManager = new ObjectManager($this); +- +- $this->objectManagerMock = $this->getMockBuilder('Magento\Framework\ObjectManagerInterface') +- ->disableOriginalConstructor() +- ->getMock(); +- $this->versionTablesrMock = $this->getMockBuilder('Magento\CatalogStaging\Model\VersionTables') ++ $this->objectManagerMock = $this->getMockBuilder(\Magento\Framework\ObjectManagerInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->model = $objectManager->getObject( + SubscriptionFactory::class, + [ + 'objectManager' => $this->objectManagerMock, +- 'versionTables' => $this->versionTablesrMock ++ 'subscriptionModels' => [ ++ 'catalog_product_entity_int' => 'ProductEntityIntSubscription' ++ ] + ] + ); + } +- + public function testCreate() + { + $data = ['tableName' => 'catalog_product_entity_int', 'columnName' => 'entity_id']; +- $versionTables = ['catalog_product_entity_int']; +- + $expectedData = $data; +- $expectedData['columnName'] = 'row_id'; +- +- $this->versionTablesrMock->expects($this->once()) +- ->method('getVersionTables') +- ->willReturn($versionTables); +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') ++ $expectedData['columnName'] = 'entity_id'; ++ $subscriptionMock = $this->getMockBuilder(\Magento\Framework\Mview\View\SubscriptionInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->objectManagerMock->expects($this->once()) + ->method('create') +- ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) ++ ->with('ProductEntityIntSubscription', $expectedData) + ->willReturn($subscriptionMock); +- + $result = $this->model->create($data); + $this->assertEquals($subscriptionMock, $result); + } +- + public function testCreateNoTableName() + { + $data = ['columnName' => 'entity_id']; +- +- $expectedData = $data; +- +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') +- ->disableOriginalConstructor() +- ->getMock(); +- $this->objectManagerMock->expects($this->once()) +- ->method('create') +- ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) +- ->willReturn($subscriptionMock); +- +- $result = $this->model->create($data); +- $this->assertEquals($subscriptionMock, $result); +- } +- +- /** +- * @param $stagingEntityTable +- * @dataProvider tablesDataProvider +- */ +- public function testCreateStagingEntityTables($stagingEntityTable) +- { +- $data = ['tableName' => $stagingEntityTable, 'columnName' => 'entity_id']; +- + $expectedData = $data; +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') ++ $subscriptionMock = $this->getMockBuilder(\Magento\Framework\Mview\View\SubscriptionInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->objectManagerMock->expects($this->once()) + ->method('create') + ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) + ->willReturn($subscriptionMock); +- +- $result = $this->model->create($data); +- $this->assertEquals($subscriptionMock, $result); +- } +- +- public static function tablesDataProvider() +- { +- return [ +- ['catalog_product_entity'], +- ['catalog_category_entity'] +- ]; +- } +- +- public function testCreateNoVersionTable() +- { +- $data = ['tableName' => 'not_existed_table', 'columnName' => 'entity_id']; +- $versionTables = ['catalog_product_entity_int']; +- +- $expectedData = $data; +- +- $this->versionTablesrMock->expects($this->once()) +- ->method('getVersionTables') +- ->willReturn($versionTables); +- $subscriptionMock = $this->getMockBuilder('Magento\Framework\Mview\View\SubscriptionInterface') +- ->disableOriginalConstructor() +- ->getMock(); +- $this->objectManagerMock->expects($this->once()) +- ->method('create') +- ->with(FrameworkSubstrictionFactory::INSTANCE_NAME, $expectedData) +- ->willReturn($subscriptionMock); +- + $result = $this->model->create($data); + $this->assertEquals($subscriptionMock, $result); + } +diff -Nuar a/vendor/magento/module-catalog-staging/etc/di.xml b/vendor/magento/module-catalog-staging/etc/di.xml +index aea000b42f..178b0bcf63 100644 +--- a/vendor/magento/module-catalog-staging/etc/di.xml ++++ b/vendor/magento/module-catalog-staging/etc/di.xml +@@ -56,6 +56,35 @@ + + + ++ ++ ++ Magento\Catalog\Api\Data\CategoryInterface ++ ++ ++ ++ ++ Magento\Catalog\Api\Data\ProductInterface ++ ++ ++ ++ ++ ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedCategoryAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ stagedProductAttributeSubscription ++ ++ ++ + + + +diff -Nuar a/vendor/magento/module-catalog-staging/etc/mview.xml b/vendor/magento/module-catalog-staging/etc/mview.xml +deleted file mode 100644 +index 45bb6589b6..0000000000 +--- a/vendor/magento/module-catalog-staging/etc/mview.xml ++++ /dev/null +@@ -1,23 +0,0 @@ +- +- +- +- +- +-
+-
+-
+-
+-
+- +- +- +- +-
+- +- +- +commit 0193f89517fc864c9ab5acd4b518f03b2c796a2f +Author: Viktor Paladiichuk +Date: Tue Nov 28 15:45:17 2017 +0200 + + MAGETWO-84444: Mview does not work with Staging + +diff -Nuar a/vendor/magento/module-catalog-staging/Model/Mview/View/Attribute/Subscription.php b/vendor/magento/module-catalog-staging/Model/Mview/View/Attribute/Subscription.php +new file mode 100644 +index 0000000000..7c549538c7 +--- /dev/null ++++ b/vendor/magento/module-catalog-staging/Model/Mview/View/Attribute/Subscription.php +@@ -0,0 +1,100 @@ ++entityMetadata = $metadataPool->getMetadata($entityInterface); ++ } ++ ++ /** ++ * Build trigger statement for INSERT, UPDATE, DELETE events ++ * ++ * @param string $event ++ * @param \Magento\Framework\Mview\View\ChangelogInterface $changelog ++ * @return string ++ */ ++ protected function buildStatement($event, $changelog) ++ { ++ $triggerBody = null; ++ switch ($event) { ++ case Trigger::EVENT_INSERT: ++ case Trigger::EVENT_UPDATE: ++ $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = NEW.%5\$s;"; ++ break; ++ case Trigger::EVENT_DELETE: ++ $triggerBody = "INSERT IGNORE INTO %1\$s (%2\$s) SELECT %3\$s FROM %4\$s WHERE %5\$s = OLD.%5\$s;"; ++ break; ++ default: ++ break; ++ } ++ $params = [ ++ $this->connection->quoteIdentifier( ++ $this->resource->getTableName($changelog->getName()) ++ ), ++ $this->connection->quoteIdentifier( ++ $changelog->getColumnName() ++ ), ++ $this->connection->quoteIdentifier( ++ $this->entityMetadata->getIdentifierField() ++ ), ++ $this->connection->quoteIdentifier( ++ $this->resource->getTableName($this->entityMetadata->getEntityTable()) ++ ), ++ $this->connection->quoteIdentifier( ++ $this->entityMetadata->getLinkField() ++ ) ++ ]; ++ return vsprintf($triggerBody, $params); ++ } ++} +commit a0dc6a745002eacaaf2ef57e37a25f79f65de650 +Author: Viktor Paladiichuk +Date: Tue Nov 28 15:54:17 2017 +0200 + + MAGETWO-84444: Mview does not work with Staging + +diff -Nuar a/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/Attribute/SubscriptionTest.php b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/Attribute/SubscriptionTest.php +new file mode 100644 +index 0000000000..d396829ef8 +--- /dev/null ++++ b/vendor/magento/module-catalog-staging/Test/Unit/Model/Mview/View/Attribute/SubscriptionTest.php +@@ -0,0 +1,302 @@ ++connectionMock = $this->getMock(Mysql::class, [], [], '', false); ++ $this->resourceMock = $this->getMock(ResourceConnection::class, [], [], '', false, false); ++ $this->connectionMock->expects($this->any()) ++ ->method('quoteIdentifier') ++ ->will($this->returnArgument(0)); ++ $this->resourceMock->expects($this->atLeastOnce()) ++ ->method('getConnection') ++ ->willReturn($this->connectionMock); ++ $this->triggerFactoryMock = $this->getMock(TriggerFactory::class, [], [], '', false, false); ++ $this->viewCollectionMock = $this->getMockForAbstractClass( ++ CollectionInterface::class, ++ [], ++ '', ++ false, ++ false, ++ true, ++ [] ++ ); ++ $this->viewMock = $this->getMockForAbstractClass(ViewInterface::class, [], '', false, false, true, []); ++ $this->resourceMock->expects($this->any()) ++ ->method('getTableName') ++ ->will($this->returnArgument(0)); ++ ++ $entityInterface = 'EntityInterface'; ++ $this->entityMetadataPoolMock = $this->getMock(MetadataPool::class, [], [], '', false); ++ ++ $this->entityMetadataMock = $this->getMock(EntityMetadataInterface::class, [], [], '', false); ++ $this->entityMetadataMock->expects($this->any()) ++ ->method('getEntityTable') ++ ->will($this->returnValue('entity_table')); ++ ++ $this->entityMetadataMock->expects($this->any()) ++ ->method('getIdentifierField') ++ ->will($this->returnValue('entity_identifier')); ++ ++ $this->entityMetadataMock->expects($this->any()) ++ ->method('getLinkField') ++ ->will($this->returnValue('entity_link_field')); ++ ++ $this->entityMetadataPoolMock->expects($this->any()) ++ ->method('getMetadata') ++ ->with($entityInterface) ++ ->will($this->returnValue($this->entityMetadataMock)); ++ ++ $this->model = new SubscriptionModel( ++ $this->resourceMock, ++ $this->triggerFactoryMock, ++ $this->viewCollectionMock, ++ $this->viewMock, ++ $this->tableName, ++ 'columnName', ++ $this->entityMetadataPoolMock, ++ $entityInterface ++ ); ++ } ++ ++ /** ++ * Prepare trigger mock ++ * ++ * @param string $triggerName ++ * @return \PHPUnit_Framework_MockObject_MockObject ++ */ ++ protected function prepareTriggerMock($triggerName) ++ { ++ $triggerMock = $this->getMockBuilder(\Magento\Framework\DB\Ddl\Trigger::class) ++ ->setMethods(['setName', 'getName', 'setTime', 'setEvent', 'setTable', 'addStatement']) ++ ->disableOriginalConstructor() ++ ->getMock(); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setName') ++ ->with($triggerName) ++ ->will($this->returnSelf()); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('getName') ++ ->will($this->returnValue('triggerName')); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setTime') ++ ->with(\Magento\Framework\DB\Ddl\Trigger::TIME_AFTER) ++ ->will($this->returnSelf()); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setEvent') ++ ->will($this->returnSelf()); ++ $triggerMock->expects($this->exactly(3)) ++ ->method('setTable') ++ ->with($this->tableName) ++ ->will($this->returnSelf()); ++ return $triggerMock; ++ } ++ ++ /** ++ * Prepare expected trigger call map ++ * ++ * @param \PHPUnit_Framework_MockObject_MockObject $triggerMock ++ * @return \PHPUnit_Framework_MockObject_MockObject ++ */ ++ protected function prepareTriggerTestCallMap(\PHPUnit_Framework_MockObject_MockObject $triggerMock) ++ { ++ $triggerMock->expects($this->at(4)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ ) ++ ->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(5)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO other_test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(11)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(12)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO other_test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = NEW.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(18)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = OLD.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ $triggerMock->expects($this->at(19)) ++ ->method('addStatement') ++ ->with( ++ "INSERT IGNORE INTO other_test_view_cl (entity_id) " ++ . "SELECT entity_identifier FROM entity_table WHERE entity_link_field = OLD.entity_link_field;" ++ )->will($this->returnSelf()); ++ ++ return $triggerMock; ++ } ++ ++ /** ++ * Prepare changelog mock ++ * ++ * @param string $changelogName ++ * @return \PHPUnit_Framework_MockObject_MockObject ++ */ ++ protected function prepareChangelogMock($changelogName) ++ { ++ $changelogMock = $this->getMockForAbstractClass( ++ \Magento\Framework\Mview\View\ChangelogInterface::class, ++ [], ++ '', ++ false, ++ false, ++ true, ++ [] ++ ); ++ $changelogMock->expects($this->exactly(3)) ++ ->method('getName') ++ ->will($this->returnValue($changelogName)); ++ $changelogMock->expects($this->exactly(3)) ++ ->method('getColumnName') ++ ->will($this->returnValue('entity_id')); ++ return $changelogMock; ++ } ++ ++ public function testCreate() ++ { ++ $triggerName = 'trigger_name'; ++ $this->resourceMock->expects($this->atLeastOnce())->method('getTriggerName')->willReturn($triggerName); ++ $triggerMock = $this->prepareTriggerMock($triggerName); ++ $this->prepareTriggerTestCallMap($triggerMock); ++ $changelogMock = $this->prepareChangelogMock('test_view_cl'); ++ ++ $this->viewMock->expects($this->exactly(3)) ++ ->method('getChangelog') ++ ->will($this->returnValue($changelogMock)); ++ ++ $this->triggerFactoryMock->expects($this->exactly(3)) ++ ->method('create') ++ ->will($this->returnValue($triggerMock)); ++ ++ $otherChangelogMock = $this->prepareChangelogMock('other_test_view_cl'); ++ ++ $otherViewMock = $this->getMockForAbstractClass( ++ ViewInterface::class, ++ [], ++ '', ++ false, ++ false, ++ true, ++ [] ++ ); ++ $otherViewMock->expects($this->exactly(1)) ++ ->method('getId') ++ ->will($this->returnValue('other_id')); ++ $otherViewMock->expects($this->exactly(1)) ++ ->method('getSubscriptions') ++ ->will($this->returnValue([['name' => $this->tableName], ['name' => 'otherTableName']])); ++ $otherViewMock->expects($this->any()) ++ ->method('getChangelog') ++ ->will($this->returnValue($otherChangelogMock)); ++ ++ $this->viewMock->expects($this->exactly(3)) ++ ->method('getId') ++ ->will($this->returnValue('this_id')); ++ $this->viewMock->expects($this->never()) ++ ->method('getSubscriptions'); ++ ++ $this->viewCollectionMock->expects($this->exactly(1)) ++ ->method('getViewsByStateMode') ++ ->with(StateInterface::MODE_ENABLED) ++ ->will($this->returnValue([$this->viewMock, $otherViewMock])); ++ ++ $this->connectionMock->expects($this->exactly(3)) ++ ->method('dropTrigger') ++ ->with('triggerName') ++ ->will($this->returnValue(true)); ++ $this->connectionMock->expects($this->exactly(3)) ++ ->method('createTrigger') ++ ->with($triggerMock); ++ ++ $this->model->create(); ++ } ++} diff --git a/patches/MAGETWO-84507__fix_complex_folder_js_bundling__2.2.0.patch b/patches/MAGETWO-84507__fix_complex_folder_js_bundling__2.2.0.patch new file mode 100644 index 0000000..7d1ab62 --- /dev/null +++ b/patches/MAGETWO-84507__fix_complex_folder_js_bundling__2.2.0.patch @@ -0,0 +1,13 @@ +diff -Nuar a/vendor/magento/module-require-js/Model/FileManager.php b/vendor/magento/module-require-js/Model/FileManager.php +--- a/vendor/magento/module-require-js/Model/FileManager.php ++++ b/vendor/magento/module-require-js/Model/FileManager.php +@@ -183,6 +183,9 @@ + } + + foreach ($libDir->read($bundleDir) as $bundleFile) { ++ if (pathinfo($bundleFile, PATHINFO_EXTENSION) !== 'js') { ++ continue; ++ } + $relPath = $libDir->getRelativePath($bundleFile); + $bundles[] = $this->assetRepo->createArbitrary($relPath, ''); + } diff --git a/patches/MAGETWO-88336__fix_complex_folder_js_bundling__2.1.4.patch b/patches/MAGETWO-88336__fix_complex_folder_js_bundling__2.1.4.patch new file mode 100644 index 0000000..2d9d8b4 --- /dev/null +++ b/patches/MAGETWO-88336__fix_complex_folder_js_bundling__2.1.4.patch @@ -0,0 +1,13 @@ +diff -Nuar a/vendor/magento/module-require-js/Model/FileManager.php b/vendor/magento/module-require-js/Model/FileManager.php +--- a/vendor/magento/module-require-js/Model/FileManager.php ++++ b/vendor/magento/module-require-js/Model/FileManager.php +@@ -164,6 +164,9 @@ + } + + foreach ($libDir->read($bundleDir) as $bundleFile) { ++ if (pathinfo($bundleFile, PATHINFO_EXTENSION) !== 'js') { ++ continue; ++ } + $relPath = $libDir->getRelativePath($bundleFile); + $bundles[] = $this->assetRepo->createArbitrary($relPath, ''); + } diff --git a/patches/MAGETWO-93265__fix_depth_of_recursive_check_of_directory_permissions__2.1.4.patch b/patches/MAGETWO-93265__fix_depth_of_recursive_check_of_directory_permissions__2.1.4.patch new file mode 100644 index 0000000..3113cb2 --- /dev/null +++ b/patches/MAGETWO-93265__fix_depth_of_recursive_check_of_directory_permissions__2.1.4.patch @@ -0,0 +1,20 @@ +diff -Nuar a/vendor/magento/framework/Setup/FilePermissions.php b/vendor/magento/framework/Setup/FilePermissions.php +--- a/vendor/magento/framework/Setup/FilePermissions.php ++++ b/vendor/magento/framework/Setup/FilePermissions.php +@@ -137,6 +137,7 @@ class FilePermissions + */ + private function checkRecursiveDirectories($directory) + { ++ /** @var $directoryIterator \RecursiveIteratorIterator */ + $directoryIterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($directory, \RecursiveDirectoryIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST +@@ -155,6 +156,8 @@ class FilePermissions + ] + ); + ++ $directoryIterator->setMaxDepth(1); ++ + $foundNonWritable = false; + + try { diff --git a/patches/MAGETWO-98833__turn_off_google_chart_api__2.x.patch b/patches/MAGETWO-98833__turn_off_google_chart_api__2.x.patch new file mode 100644 index 0000000..4d0dbd8 --- /dev/null +++ b/patches/MAGETWO-98833__turn_off_google_chart_api__2.x.patch @@ -0,0 +1,141 @@ +diff --git a/vendor/magento/module-backend/Block/Dashboard/Graph.php b/vendor/magento/module-backend/Block/Dashboard/Graph.php +index 8e238ccab44c..71a6cf4e938f 100644 +--- a/vendor/magento/module-backend/Block/Dashboard/Graph.php ++++ b/vendor/magento/module-backend/Block/Dashboard/Graph.php +@@ -15,7 +15,7 @@ class Graph extends \Magento\Backend\Block\Dashboard\AbstractDashboard + /** + * Api URL + */ +- const API_URL = 'http://chart.apis.google.com/chart'; ++ const API_URL = 'https://image-charts.com/chart'; + + /** + * All series +@@ -76,6 +76,7 @@ class Graph extends \Magento\Backend\Block\Dashboard\AbstractDashboard + /** + * Google chart api data encoding + * ++ * @deprecated since the Google Image Charts API not accessible from March 14, 2019 + * @var string + */ + protected $_encoding = 'e'; +@@ -187,11 +188,12 @@ public function getChartUrl($directUrl = true) + { + $params = [ + 'cht' => 'lc', +- 'chf' => 'bg,s,ffffff', +- 'chco' => 'ef672f', + 'chls' => '7', +- 'chxs' => '0,676056,15,0,l,676056|1,676056,15,0,l,676056', +- 'chm' => 'h,f2ebde,0,0:1:.1,1,-1', ++ 'chf' => 'bg,s,f4f4f4|c,lg,90,ffffff,0.1,ededed,0', ++ 'chm' => 'B,f4d4b2,0,0,0', ++ 'chco' => 'db4814', ++ 'chxs' => '0,0,11|1,0,11', ++ 'chma' => '15,15,15,15' + ]; + + $this->_allSeries = $this->getRowsData($this->_dataRows); +@@ -279,20 +281,11 @@ public function getChartUrl($directUrl = true) + $this->_axisLabels['x'] = $dates; + $this->_allSeries = $datas; + +- //Google encoding values +- if ($this->_encoding == "s") { +- // simple encoding +- $params['chd'] = "s:"; +- $dataDelimiter = ""; +- $dataSetdelimiter = ","; +- $dataMissing = "_"; +- } else { +- // extended encoding +- $params['chd'] = "e:"; +- $dataDelimiter = ""; +- $dataSetdelimiter = ","; +- $dataMissing = "__"; +- } ++ // Image-Charts Awesome data format values ++ $params['chd'] = "a:"; ++ $dataDelimiter = ","; ++ $dataSetdelimiter = "|"; ++ $dataMissing = "_"; + + // process each string in the array, and find the max length + $localmaxvalue = [0]; +@@ -306,7 +299,6 @@ public function getChartUrl($directUrl = true) + $minvalue = min($localminvalue); + + // default values +- $yrange = 0; + $yLabels = []; + $miny = 0; + $maxy = 0; +@@ -321,7 +313,6 @@ public function getChartUrl($directUrl = true) + $maxy = ceil($maxvalue + 1); + $yLabels = range($miny, $maxy, 1); + } +- $yrange = $maxy; + $yorigin = 0; + } + +@@ -329,44 +320,13 @@ public function getChartUrl($directUrl = true) + + foreach ($this->getAllSeries() as $index => $serie) { + $thisdataarray = $serie; +- if ($this->_encoding == "s") { +- // SIMPLE ENCODING +- for ($j = 0; $j < sizeof($thisdataarray); $j++) { +- $currentvalue = $thisdataarray[$j]; +- if (is_numeric($currentvalue)) { +- $ylocation = round( +- (strlen($this->_simpleEncoding) - 1) * ($yorigin + $currentvalue) / $yrange +- ); +- $chartdata[] = substr($this->_simpleEncoding, $ylocation, 1) . $dataDelimiter; +- } else { +- $chartdata[] = $dataMissing . $dataDelimiter; +- } +- } +- } else { +- // EXTENDED ENCODING +- for ($j = 0; $j < sizeof($thisdataarray); $j++) { +- $currentvalue = $thisdataarray[$j]; +- if (is_numeric($currentvalue)) { +- if ($yrange) { +- $ylocation = 4095 * ($yorigin + $currentvalue) / $yrange; +- } else { +- $ylocation = 0; +- } +- $firstchar = floor($ylocation / 64); +- $secondchar = $ylocation % 64; +- $mappedchar = substr( +- $this->_extendedEncoding, +- $firstchar, +- 1 +- ) . substr( +- $this->_extendedEncoding, +- $secondchar, +- 1 +- ); +- $chartdata[] = $mappedchar . $dataDelimiter; +- } else { +- $chartdata[] = $dataMissing . $dataDelimiter; +- } ++ for ($j = 0; $j < sizeof($thisdataarray); $j++) { ++ $currentvalue = $thisdataarray[$j]; ++ if (is_numeric($currentvalue)) { ++ $ylocation = $yorigin + $currentvalue; ++ $chartdata[] = $ylocation . $dataDelimiter; ++ } else { ++ $chartdata[] = $dataMissing . $dataDelimiter; + } + } + $chartdata[] = $dataSetdelimiter; +@@ -540,6 +500,8 @@ protected function getHeight() + } + + /** ++ * Sets data helper ++ * + * @param \Magento\Backend\Helper\Dashboard\AbstractDashboard $dataHelper + * @return void + */ diff --git a/patches/MC-5964__preauth_sql__2.1.4.patch b/patches/MC-5964__preauth_sql__2.1.4.patch new file mode 100644 index 0000000..78a189b --- /dev/null +++ b/patches/MC-5964__preauth_sql__2.1.4.patch @@ -0,0 +1,12 @@ +diff -Naur a/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php b/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php +--- a/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php ++++ b/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php +@@ -2955,7 +2955,7 @@ class Mysql extends \Zend_Db_Adapter_Pdo_Mysql implements AdapterInterface + if (isset($condition['to'])) { + $query .= empty($query) ? '' : ' AND '; + $to = $this->_prepareSqlDateCondition($condition, 'to'); +- $query = $this->_prepareQuotedSqlCondition($query . $conditionKeyMap['to'], $to, $fieldName); ++ $query = $query . $this->_prepareQuotedSqlCondition($conditionKeyMap['to'], $to, $fieldName); + } + } elseif (array_key_exists($key, $conditionKeyMap)) { + $value = $condition[$key]; diff --git a/patches/MC-5964__preauth_sql__2.2.0.patch b/patches/MC-5964__preauth_sql__2.2.0.patch new file mode 100644 index 0000000..82a111e --- /dev/null +++ b/patches/MC-5964__preauth_sql__2.2.0.patch @@ -0,0 +1,92 @@ +diff -Naur a/vendor/magento/module-catalog/Model/Product/ProductFrontendAction/Synchronizer.php b/vendor/magento/module-catalog/Model/Product/ProductFrontendAction/Synchronizer.php +--- a/vendor/magento/module-catalog/Model/Product/ProductFrontendAction/Synchronizer.php ++++ b/vendor/magento/module-catalog/Model/Product/ProductFrontendAction/Synchronizer.php +@@ -138,7 +138,9 @@ private function getProductIdsByActions(array $actions) + $productIds = []; + + foreach ($actions as $action) { +- $productIds[] = $action['product_id']; ++ if (isset($action['product_id']) && is_int($action['product_id'])) { ++ $productIds[] = $action['product_id']; ++ } + } + + return $productIds; +@@ -159,33 +161,37 @@ public function syncActions(array $productsData, $typeId) + $customerId = $this->session->getCustomerId(); + $visitorId = $this->visitor->getId(); + $collection = $this->getActionsByType($typeId); +- $collection->addFieldToFilter('product_id', $this->getProductIdsByActions($productsData)); +- +- /** +- * Note that collection is also filtered by visitor id and customer id +- * This collection shouldnt be flushed when visitor has products and then login +- * It can remove only products for visitor, or only products for customer +- * +- * ['product_id' => 'added_at'] +- * @var ProductFrontendActionInterface $item +- */ +- foreach ($collection as $item) { +- $this->entityManager->delete($item); +- } +- +- foreach ($productsData as $productId => $productData) { +- /** @var ProductFrontendActionInterface $action */ +- $action = $this->productFrontendActionFactory->create([ +- 'data' => [ +- 'visitor_id' => $customerId ? null : $visitorId, +- 'customer_id' => $this->session->getCustomerId(), +- 'added_at' => $productData['added_at'], +- 'product_id' => $productId, +- 'type_id' => $typeId +- ] +- ]); +- +- $this->entityManager->save($action); ++ $productIds = $this->getProductIdsByActions($productsData); ++ ++ if ($productIds) { ++ $collection->addFieldToFilter('product_id', $productIds); ++ ++ /** ++ * Note that collection is also filtered by visitor id and customer id ++ * This collection shouldnt be flushed when visitor has products and then login ++ * It can remove only products for visitor, or only products for customer ++ * ++ * ['product_id' => 'added_at'] ++ * @var ProductFrontendActionInterface $item ++ */ ++ foreach ($collection as $item) { ++ $this->entityManager->delete($item); ++ } ++ ++ foreach ($productsData as $productId => $productData) { ++ /** @var ProductFrontendActionInterface $action */ ++ $action = $this->productFrontendActionFactory->create([ ++ 'data' => [ ++ 'visitor_id' => $customerId ? null : $visitorId, ++ 'customer_id' => $this->session->getCustomerId(), ++ 'added_at' => $productData['added_at'], ++ 'product_id' => $productId, ++ 'type_id' => $typeId ++ ] ++ ]); ++ ++ $this->entityManager->save($action); ++ } + } + } + +diff -Naur a/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php b/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php +index 3d06e27542f0..a6c0dba6e175 100644 +--- a/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php ++++ b/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php +@@ -2904,7 +2904,7 @@ public function prepareSqlCondition($fieldName, $condition) + if (isset($condition['to'])) { + $query .= empty($query) ? '' : ' AND '; + $to = $this->_prepareSqlDateCondition($condition, 'to'); +- $query = $this->_prepareQuotedSqlCondition($query . $conditionKeyMap['to'], $to, $fieldName); ++ $query = $query . $this->_prepareQuotedSqlCondition($conditionKeyMap['to'], $to, $fieldName); + } + } elseif (array_key_exists($key, $conditionKeyMap)) { + $value = $condition[$key]; diff --git a/patches/MC-5964__preauth_sql__2.3.0.patch b/patches/MC-5964__preauth_sql__2.3.0.patch new file mode 100644 index 0000000..fe7b2cf --- /dev/null +++ b/patches/MC-5964__preauth_sql__2.3.0.patch @@ -0,0 +1,123 @@ +diff -Naur a/vendor/magento/module-catalog/Model/Product/ProductFrontendAction/Synchronizer.php b/vendor/magento/module-catalog/Model/Product/ProductFrontendAction/Synchronizer.php +--- a/vendor/magento/module-catalog/Model/Product/ProductFrontendAction/Synchronizer.php ++++ b/vendor/magento/module-catalog/Model/Product/ProductFrontendAction/Synchronizer.php +@@ -16,6 +16,8 @@ + use Magento\Framework\EntityManager\EntityManager; + + /** ++ * A Product Widget Synchronizer. ++ * + * Service which allows to sync product widget information, such as product id with db. In order to reuse this info + * on different devices + */ +@@ -85,9 +87,10 @@ public function __construct( + } + + /** +- * Find lifetime in configuration. Configuration is hold in Stores Configuration +- * Also this configuration is generated by: +- * @see \Magento\Catalog\Model\Widget\RecentlyViewedStorageConfiguration ++ * Finds lifetime in configuration. ++ * ++ * Configuration is hold in Stores Configuration. Also this configuration is generated by ++ * {@see Magento\Catalog\Model\Widget\RecentlyViewedStorageConfiguration} + * + * @param string $namespace + * @return int +@@ -108,6 +111,8 @@ private function getLifeTimeByNamespace($namespace) + } + + /** ++ * Filters actions. ++ * + * In order to avoid suspicious actions, we need to filter them in DESC order, and slice only items that + * can be persisted in database. + * +@@ -138,7 +143,9 @@ private function getProductIdsByActions(array $actions) + $productIds = []; + + foreach ($actions as $action) { +- $productIds[] = $action['product_id']; ++ if (isset($action['product_id']) && is_int($action['product_id'])) { ++ $productIds[] = $action['product_id']; ++ } + } + + return $productIds; +@@ -159,33 +166,37 @@ public function syncActions(array $productsData, $typeId) + $customerId = $this->session->getCustomerId(); + $visitorId = $this->visitor->getId(); + $collection = $this->getActionsByType($typeId); +- $collection->addFieldToFilter('product_id', $this->getProductIdsByActions($productsData)); +- +- /** +- * Note that collection is also filtered by visitor id and customer id +- * This collection shouldn't be flushed when visitor has products and then login +- * It can remove only products for visitor, or only products for customer +- * +- * ['product_id' => 'added_at'] +- * @var ProductFrontendActionInterface $item +- */ +- foreach ($collection as $item) { +- $this->entityManager->delete($item); +- } +- +- foreach ($productsData as $productId => $productData) { +- /** @var ProductFrontendActionInterface $action */ +- $action = $this->productFrontendActionFactory->create([ +- 'data' => [ +- 'visitor_id' => $customerId ? null : $visitorId, +- 'customer_id' => $this->session->getCustomerId(), +- 'added_at' => $productData['added_at'], +- 'product_id' => $productId, +- 'type_id' => $typeId +- ] +- ]); +- +- $this->entityManager->save($action); ++ $productIds = $this->getProductIdsByActions($productsData); ++ ++ if ($productIds) { ++ $collection->addFieldToFilter('product_id', $productIds); ++ ++ /** ++ * Note that collection is also filtered by visitor id and customer id ++ * This collection shouldn't be flushed when visitor has products and then login ++ * It can remove only products for visitor, or only products for customer ++ * ++ * ['product_id' => 'added_at'] ++ * @var ProductFrontendActionInterface $item ++ */ ++ foreach ($collection as $item) { ++ $this->entityManager->delete($item); ++ } ++ ++ foreach ($productsData as $productId => $productData) { ++ /** @var ProductFrontendActionInterface $action */ ++ $action = $this->productFrontendActionFactory->create([ ++ 'data' => [ ++ 'visitor_id' => $customerId ? null : $visitorId, ++ 'customer_id' => $this->session->getCustomerId(), ++ 'added_at' => $productData['added_at'], ++ 'product_id' => $productId, ++ 'type_id' => $typeId ++ ] ++ ]); ++ ++ $this->entityManager->save($action); ++ } + } + } + +diff -Naur a/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php b/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php +--- a/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php ++++ b/vendor/magento/framework/DB/Adapter/Pdo/Mysql.php +@@ -2955,7 +2955,7 @@ class Mysql extends \Zend_Db_Adapter_Pdo_Mysql implements AdapterInterface + if (isset($condition['to'])) { + $query .= empty($query) ? '' : ' AND '; + $to = $this->_prepareSqlDateCondition($condition, 'to'); +- $query = $this->_prepareQuotedSqlCondition($query . $conditionKeyMap['to'], $to, $fieldName); ++ $query = $query . $this->_prepareQuotedSqlCondition($conditionKeyMap['to'], $to, $fieldName); + } + } elseif (array_key_exists($key, $conditionKeyMap)) { + $value = $condition[$key]; diff --git a/patches/MDVA-2470__fix_asset_locking_race_condition__2.1.4.patch b/patches/MDVA-2470__fix_asset_locking_race_condition__2.1.4.patch new file mode 100644 index 0000000..0152839 --- /dev/null +++ b/patches/MDVA-2470__fix_asset_locking_race_condition__2.1.4.patch @@ -0,0 +1,109 @@ +MDVA-2470 +diff -Naur a/vendor/magento/module-deploy/Model/Deploy/LocaleDeploy.php b/vendor/magento/module-deploy/Model/Deploy/LocaleDeploy.php +--- a/vendor/magento/module-deploy/Model/Deploy/LocaleDeploy.php 2017-02-08 19:50:59.000000000 +0000 ++++ b/vendor/magento/module-deploy/Model/Deploy/LocaleDeploy.php 2017-02-13 18:50:46.000000000 +0000 +@@ -410,7 +410,7 @@ + $this->logger->critical($errorMessage); + } catch (\Exception $exception) { + $this->output->write('.'); +- if ($this->output->isVerbose()) { ++ if ($this->output->isVerbose() || $this->output->isVeryVerbose()) { + $this->output->writeln($exception->getTraceAsString()); + } + $this->errorCount++; + +diff -Naur a/vendor/magento/framework/View/Asset/LockerProcess.php b/vendor/magento/framework/View/Asset/LockerProcess.php +--- a/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-13 17:38:15.000000000 +0000 ++++ b/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-13 18:53:22.000000000 +0000 +@@ -68,12 +68,26 @@ + + $this->tmpDirectory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); + $this->lockFilePath = $this->getFilePath($lockName); ++ $this->waitForLock(); + +- while ($this->isProcessLocked()) { +- usleep(1000); ++ try { ++ $this->tmpDirectory->writeFile($this->lockFilePath, time(), 'x+'); ++ }catch (\Exception $e) { ++ $this->waitForLock(); ++ try { ++ $this->tmpDirectory->writeFile($this->lockFilePath, time(), 'x+'); ++ }catch (\Exception $e) { ++ throw new \Exception($e->getMessage()); ++ } + } + +- $this->tmpDirectory->writeFile($this->lockFilePath, time()); ++ } ++ ++ public function waitForLock() ++ { ++ while ($this->isProcessLocked() ) { ++ usleep(500); ++ } + } + + /** + +diff -Nuar a/vendor/magento/module-deploy/Model/Deploy/LocaleDeploy.php b/vendor/magento/module-deploy/Model/Deploy/LocaleDeploy.php +--- a/vendor/magento/module-deploy/Model/Deploy/LocaleDeploy.php 2017-02-14 20:20:28.000000000 +0000 ++++ b/vendor/magento/module-deploy/Model/Deploy/LocaleDeploy.php 2017-02-14 20:40:07.000000000 +0000 +@@ -410,9 +410,7 @@ + $this->logger->critical($errorMessage); + } catch (\Exception $exception) { + $this->output->write('.'); +- if ($this->output->isVerbose() || $this->output->isVeryVerbose()) { +- $this->output->writeln($exception->getTraceAsString()); +- } ++ $this->output->writeln($exception->getTraceAsString()); + $this->errorCount++; + } + +diff -Naur a/vendor/magento/framework/View/Asset/PreProcessor/AlternativeSource.php b/vendor/magento/framework/View/Asset/PreProcessor/AlternativeSource.php +--- a/vendor/magento/framework/View/Asset/PreProcessor/AlternativeSource.php 2017-02-14 20:49:33.000000000 +0000 ++++ b/vendor/magento/framework/View/Asset/PreProcessor/AlternativeSource.php 2017-02-15 15:00:41.000000000 +0000 +@@ -106,7 +106,7 @@ + } + + try { +- $this->lockerProcess->lockProcess($this->lockName); ++ $this->lockerProcess->lockProcess($chain->getAsset()->getPath()); + + $module = $chain->getAsset()->getModule(); + +diff -Naur a/vendor/magento/framework/View/Asset/LockerProcess.php b/vendor/magento/framework/View/Asset/LockerProcess.php +--- a/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-14 21:50:57.000000000 +0000 ++++ b/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-15 15:00:41.000000000 +0000 +@@ -67,7 +67,7 @@ + } + + $this->tmpDirectory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); +- $this->lockFilePath = $this->getFilePath($lockName); ++ $this->lockFilePath = $this->getFilePath(str_replace(DIRECTORY_SEPARATOR, '_', $lockName)); + $this->waitForLock(); + + try { +@@ -77,7 +77,8 @@ + try { + $this->tmpDirectory->writeFile($this->lockFilePath, time(), 'x+'); + }catch (\Exception $e) { +- throw new \Exception($e->getMessage()); ++ echo($this->lockFilePath); ++ throw new \Exception("In exception for lock process" . $e->getMessage()); + } + } + +diff -Naur a/vendor/magento/module-developer/Model/View/Asset/PreProcessor/FrontendCompilation.php b/vendor/magento/module-developer/Model/View/Asset/PreProcessor/FrontendCompilation.php +--- a/vendor/magento/module-developer/Model/View/Asset/PreProcessor/FrontendCompilation.php 2017-02-15 16:24:07.000000000 +0000 ++++ b/vendor/magento/module-developer/Model/View/Asset/PreProcessor/FrontendCompilation.php 2017-02-15 16:24:07.000000000 +0000 +@@ -80,7 +80,7 @@ + } + + try { +- $this->lockerProcess->lockProcess($this->lockName); ++ $this->lockerProcess->lockProcess($chain->getAsset()->getPath()); + + $path = $chain->getAsset()->getFilePath(); + $module = $chain->getAsset()->getModule(); + diff --git a/patches/MDVA-2470__fix_asset_locking_race_condition__2.2.0.patch b/patches/MDVA-2470__fix_asset_locking_race_condition__2.2.0.patch new file mode 100644 index 0000000..ed82a40 --- /dev/null +++ b/patches/MDVA-2470__fix_asset_locking_race_condition__2.2.0.patch @@ -0,0 +1,80 @@ +MDVA-2470 +diff -Nuar a/vendor/magento/framework/View/Asset/LockerProcess.php b/vendor/magento/framework/View/Asset/LockerProcess.php +--- a/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-13 17:38:15.000000000 +0000 ++++ b/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-13 18:53:22.000000000 +0000 +@@ -68,12 +68,26 @@ + + $this->tmpDirectory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); + $this->lockFilePath = $this->getFilePath($lockName); ++ $this->waitForLock(); + +- while ($this->isProcessLocked()) { +- usleep(1000); ++ try { ++ $this->tmpDirectory->writeFile($this->lockFilePath, time(), 'x+'); ++ }catch (\Exception $e) { ++ $this->waitForLock(); ++ try { ++ $this->tmpDirectory->writeFile($this->lockFilePath, time(), 'x+'); ++ }catch (\Exception $e) { ++ throw new \Exception($e->getMessage()); ++ } + } + +- $this->tmpDirectory->writeFile($this->lockFilePath, time()); ++ } ++ ++ public function waitForLock() ++ { ++ while ($this->isProcessLocked() ) { ++ usleep(500); ++ } + } + + /** +diff -Nuar a/vendor/magento/framework/View/Asset/PreProcessor/AlternativeSource.php b/vendor/magento/framework/View/Asset/PreProcessor/AlternativeSource.php +--- a/vendor/magento/framework/View/Asset/PreProcessor/AlternativeSource.php 2017-02-14 20:49:33.000000000 +0000 ++++ b/vendor/magento/framework/View/Asset/PreProcessor/AlternativeSource.php 2017-02-15 15:00:41.000000000 +0000 +@@ -106,7 +106,7 @@ + } + + try { +- $this->lockerProcess->lockProcess($this->lockName); ++ $this->lockerProcess->lockProcess($chain->getAsset()->getPath()); + + $module = $chain->getAsset()->getModule(); + +diff -Nuar a/vendor/magento/framework/View/Asset/LockerProcess.php b/vendor/magento/framework/View/Asset/LockerProcess.php +--- a/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-14 21:50:57.000000000 +0000 ++++ b/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-15 15:00:41.000000000 +0000 +@@ -67,7 +67,7 @@ + } + + $this->tmpDirectory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); +- $this->lockFilePath = $this->getFilePath($lockName); ++ $this->lockFilePath = $this->getFilePath(str_replace(DIRECTORY_SEPARATOR, '_', $lockName)); + $this->waitForLock(); + + try { +@@ -77,7 +77,8 @@ + try { + $this->tmpDirectory->writeFile($this->lockFilePath, time(), 'x+'); + }catch (\Exception $e) { +- throw new \Exception($e->getMessage()); ++ echo($this->lockFilePath); ++ throw new \Exception("In exception for lock process" . $e->getMessage()); + } + } + +diff -Nuar a/vendor/magento/module-developer/Model/View/Asset/PreProcessor/FrontendCompilation.php b/vendor/magento/module-developer/Model/View/Asset/PreProcessor/FrontendCompilation.php +--- a/vendor/magento/module-developer/Model/View/Asset/PreProcessor/FrontendCompilation.php 2017-02-15 16:24:07.000000000 +0000 ++++ b/vendor/magento/module-developer/Model/View/Asset/PreProcessor/FrontendCompilation.php 2017-02-15 16:24:07.000000000 +0000 +@@ -76,7 +76,7 @@ + { + + try { +- $this->lockerProcess->lockProcess($this->lockName); ++ $this->lockerProcess->lockProcess($chain->getAsset()->getPath()); + + $path = $chain->getAsset()->getFilePath(); + $module = $chain->getAsset()->getModule(); diff --git a/patches/MDVA-8695__properly_encode_characters_in_emails__2.1.4.patch b/patches/MDVA-8695__properly_encode_characters_in_emails__2.1.4.patch new file mode 100644 index 0000000..6e2036c --- /dev/null +++ b/patches/MDVA-8695__properly_encode_characters_in_emails__2.1.4.patch @@ -0,0 +1,12 @@ +diff -Nuar a/vendor/magento/framework/Mail/Message.php b/vendor/magento/framework/Mail/Message.php +index 36a0e6a..fad0910 100644 +--- a/vendor/magento/framework/Mail/Message.php ++++ b/vendor/magento/framework/Mail/Message.php +@@ -15,6 +15,7 @@ class Message extends \Zend_Mail implements MessageInterface + public function __construct($charset = 'utf-8') + { + parent::__construct($charset); ++ $this->setHeaderEncoding(\Zend_Mime::ENCODING_BASE64); + } + + /** diff --git a/patches/MSI-2210__price_indexer_fails_with_large_catalogs__1.0.3.patch b/patches/MSI-2210__price_indexer_fails_with_large_catalogs__1.0.3.patch new file mode 100644 index 0000000..976f65e --- /dev/null +++ b/patches/MSI-2210__price_indexer_fails_with_large_catalogs__1.0.3.patch @@ -0,0 +1,88 @@ +diff -Nuar a/vendor/magento/module-inventory-catalog/Plugin/CatalogInventory/Model/Indexer/ModifySelectInProductPriceIndexFilter.php b/vendor/magento/module-inventory-catalog/Plugin/CatalogInventory/Model/Indexer/ModifySelectInProductPriceIndexFilter.php +--- a/vendor/magento/module-inventory-catalog/Plugin/CatalogInventory/Model/Indexer/ModifySelectInProductPriceIndexFilter.php ++++ b/vendor/magento/module-inventory-catalog/Plugin/CatalogInventory/Model/Indexer/ModifySelectInProductPriceIndexFilter.php +@@ -11,6 +11,8 @@ + use Magento\CatalogInventory\Api\StockConfigurationInterface; + use Magento\CatalogInventory\Model\Indexer\ProductPriceIndexFilter; + use Magento\Framework\App\ResourceConnection; ++use Magento\InventoryApi\Api\Data\StockInterface; ++use Magento\InventoryCatalogApi\Api\DefaultStockProviderInterface; + use Magento\InventoryIndexer\Model\StockIndexTableNameResolverInterface; + use Magento\InventorySalesApi\Model\StockByWebsiteIdResolverInterface; + +@@ -39,22 +41,30 @@ class ModifySelectInProductPriceIndexFilter + */ + private $stockByWebsiteIdResolver; + ++ /** ++ * @var DefaultStockProviderInterface ++ */ ++ private $defaultStockProvider; ++ + /** + * @param StockIndexTableNameResolverInterface $stockIndexTableNameResolver + * @param StockConfigurationInterface $stockConfiguration + * @param ResourceConnection $resourceConnection + * @param StockByWebsiteIdResolverInterface $stockByWebsiteIdResolver ++ * @param DefaultStockProviderInterface $defaultStockProvider + */ + public function __construct( + StockIndexTableNameResolverInterface $stockIndexTableNameResolver, + StockConfigurationInterface $stockConfiguration, + ResourceConnection $resourceConnection, +- StockByWebsiteIdResolverInterface $stockByWebsiteIdResolver ++ StockByWebsiteIdResolverInterface $stockByWebsiteIdResolver, ++ DefaultStockProviderInterface $defaultStockProvider + ) { + $this->stockIndexTableNameResolver = $stockIndexTableNameResolver; + $this->stockConfiguration = $stockConfiguration; + $this->resourceConnection = $resourceConnection; + $this->stockByWebsiteIdResolver = $stockByWebsiteIdResolver; ++ $this->defaultStockProvider = $defaultStockProvider; + } + + /** +@@ -84,7 +94,8 @@ public function aroundModifyPrice( + $select->from(['price_index' => $priceTable->getTableName()], []); + $priceEntityField = $priceTable->getEntityField(); + +- if ($this->resourceConnection->getConnection()->isTableExists($stockTable)) { ++ if (!$this->isDefaultStock($stock) ++ && $this->resourceConnection->getConnection()->isTableExists($stockTable)) { + $select->joinInner( + ['product_entity' => $this->resourceConnection->getTableName('catalog_product_entity')], + "product_entity.entity_id = price_index.{$priceEntityField}", +@@ -95,6 +106,17 @@ public function aroundModifyPrice( + [] + ); + $select->where('inventory_stock.is_salable = 0 OR inventory_stock.is_salable IS NULL'); ++ } else { ++ $legacyStockTableName = $this->resourceConnection->getTableName('cataloginventory_stock_status'); ++ $select->joinLeft( ++ ['stock_status' => $legacyStockTableName], ++ sprintf( ++ 'stock_status.product_id = price_index.%s', ++ $priceEntityField ++ ), ++ [] ++ ); ++ $select->where('stock_status.stock_status = 0 OR stock_status.stock_status IS NULL'); + } + + $select->where('price_index.website_id = ?', $websiteId); +@@ -126,4 +148,15 @@ private function getWebsiteIdsFromProducts(array $entityIds): array + + return $result; + } ++ ++ /** ++ * Checks if inventory stock is DB view ++ * ++ * @param StockInterface $stock ++ * @return bool ++ */ ++ private function isDefaultStock(StockInterface $stock): bool ++ { ++ return (int)$stock->getStockId() === $this->defaultStockProvider->getId(); ++ } + } diff --git a/patches/MSI-GH-2350__avoid_quering_inventory_default_stock_view_in_storefront__1.0.3.patch b/patches/MSI-GH-2350__avoid_quering_inventory_default_stock_view_in_storefront__1.0.3.patch new file mode 100644 index 0000000..7fd1873 --- /dev/null +++ b/patches/MSI-GH-2350__avoid_quering_inventory_default_stock_view_in_storefront__1.0.3.patch @@ -0,0 +1,82 @@ +diff -Nuar a/vendor/magento/module-inventory-indexer/Model/ResourceModel/GetStockItemData.php b/vendor/magento/module-inventory-indexer/Model/ResourceModel/GetStockItemData.php +--- a/vendor/magento/module-inventory-indexer/Model/ResourceModel/GetStockItemData.php ++++ b/vendor/magento/module-inventory-indexer/Model/ResourceModel/GetStockItemData.php +@@ -12,6 +12,8 @@ + use Magento\InventoryIndexer\Model\StockIndexTableNameResolverInterface; + use Magento\InventorySalesApi\Model\GetStockItemDataInterface; + use Magento\InventoryIndexer\Indexer\IndexStructure; ++use Magento\InventoryCatalogApi\Api\DefaultStockProviderInterface; ++use Magento\InventoryCatalogApi\Model\GetProductIdsBySkusInterface; + + /** + * @inheritdoc +@@ -28,16 +30,32 @@ class GetStockItemData implements GetStockItemDataInterface + */ + private $stockIndexTableNameResolver; + ++ /** ++ * @var DefaultStockProviderInterface ++ */ ++ private $defaultStockProvider; ++ ++ /** ++ * @var GetProductIdsBySkusInterface ++ */ ++ private $getProductIdsBySkus; ++ + /** + * @param ResourceConnection $resource + * @param StockIndexTableNameResolverInterface $stockIndexTableNameResolver ++ * @param DefaultStockProviderInterface $defaultStockProvider ++ * @param GetProductIdsBySkusInterface $getProductIdsBySkus + */ + public function __construct( + ResourceConnection $resource, +- StockIndexTableNameResolverInterface $stockIndexTableNameResolver ++ StockIndexTableNameResolverInterface $stockIndexTableNameResolver, ++ DefaultStockProviderInterface $defaultStockProvider, ++ GetProductIdsBySkusInterface $getProductIdsBySkus + ) { + $this->resource = $resource; + $this->stockIndexTableNameResolver = $stockIndexTableNameResolver; ++ $this->defaultStockProvider = $defaultStockProvider; ++ $this->getProductIdsBySkus = $getProductIdsBySkus; + } + + /** +@@ -45,18 +63,29 @@ public function __construct( + */ + public function execute(string $sku, int $stockId): ?array + { +- $stockItemTableName = $this->stockIndexTableNameResolver->execute($stockId); +- + $connection = $this->resource->getConnection(); +- $select = $connection->select() +- ->from( ++ $select = $connection->select(); ++ ++ if ($this->defaultStockProvider->getId() === $stockId) { ++ $productId = current($this->getProductIdsBySkus->execute([$sku])); ++ $stockItemTableName = $this->resource->getTableName('cataloginventory_stock_status'); ++ $select->from( ++ $stockItemTableName, ++ [ ++ GetStockItemDataInterface::QUANTITY => 'qty', ++ GetStockItemDataInterface::IS_SALABLE => 'stock_status', ++ ] ++ )->where('product_id = ?', $productId); ++ } else { ++ $stockItemTableName = $this->stockIndexTableNameResolver->execute($stockId); ++ $select->from( + $stockItemTableName, + [ + GetStockItemDataInterface::QUANTITY => IndexStructure::QUANTITY, + GetStockItemDataInterface::IS_SALABLE => IndexStructure::IS_SALABLE, + ] +- ) +- ->where(IndexStructure::SKU . ' = ?', $sku); ++ )->where(IndexStructure::SKU . ' = ?', $sku); ++ } + + try { + if ($connection->isTableExists($stockItemTableName)) { diff --git a/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__configurable-product-indexer__1.0.3.patch b/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__configurable-product-indexer__1.0.3.patch new file mode 100644 index 0000000..cec0575 --- /dev/null +++ b/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__configurable-product-indexer__1.0.3.patch @@ -0,0 +1,102 @@ +diff -Nuar a/vendor/magento/module-inventory-configurable-product-indexer/Indexer/SourceItem/SiblingSkuListInStockProvider.php b/vendor/magento/module-inventory-configurable-product-indexer/Indexer/SourceItem/SiblingSkuListInStockProvider.php +--- a/vendor/magento/module-inventory-configurable-product-indexer/Indexer/SourceItem/SiblingSkuListInStockProvider.php ++++ b/vendor/magento/module-inventory-configurable-product-indexer/Indexer/SourceItem/SiblingSkuListInStockProvider.php +@@ -31,10 +31,6 @@ class SiblingSkuListInStockProvider + */ + private $skuListInStockFactory; + +- /** +- * @var int +- */ +- private $groupConcatMaxLen; + /** + * @var MetadataPool + */ +@@ -56,7 +52,6 @@ class SiblingSkuListInStockProvider + * @param ResourceConnection $resourceConnection + * @param SkuListInStockFactory $skuListInStockFactory + * @param MetadataPool $metadataPool +- * @param int $groupConcatMaxLen + * @param string $tableNameSourceItem + * @param string $tableNameStockSourceLink + */ +@@ -64,13 +59,11 @@ public function __construct( + ResourceConnection $resourceConnection, + SkuListInStockFactory $skuListInStockFactory, + MetadataPool $metadataPool, +- int $groupConcatMaxLen, + $tableNameSourceItem, + $tableNameStockSourceLink + ) { + $this->resourceConnection = $resourceConnection; + $this->skuListInStockFactory = $skuListInStockFactory; +- $this->groupConcatMaxLen = $groupConcatMaxLen; + $this->metadataPool = $metadataPool; + $this->tableNameSourceItem = $tableNameSourceItem; + $this->tableNameStockSourceLink = $tableNameStockSourceLink; +@@ -91,15 +84,13 @@ public function execute(array $sourceItemIds): array + + $metadata = $this->metadataPool->getMetadata(ProductInterface::class); + $linkField = $metadata->getLinkField(); ++ $items = []; + + $select = $connection + ->select() + ->from( + ['source_item' => $sourceItemTable], +- [ +- SourceItemInterface::SKU => +- "GROUP_CONCAT(DISTINCT sibling_product_entity." . SourceItemInterface::SKU . " SEPARATOR ',')" +- ] ++ [SourceItemInterface::SKU => 'sibling_product_entity.' . SourceItemInterface::SKU] + )->joinInner( + ['stock_source_link' => $sourceStockLinkTable], + sprintf( +@@ -124,11 +115,17 @@ public function execute(array $sourceItemIds): array + ['sibling_product_entity' => $this->resourceConnection->getTableName('catalog_product_entity')], + 'sibling_product_entity.' . $linkField . ' = sibling_link.product_id', + [] +- )->where('source_item.source_item_id IN (?)', $sourceItemIds) +- ->group(['stock_source_link.' . StockSourceLinkInterface::STOCK_ID]); ++ )->where( ++ 'source_item.source_item_id IN (?)', ++ $sourceItemIds ++ ); ++ ++ $dbStatement = $connection->query($select); ++ while ($item = $dbStatement->fetch()) { ++ $items[$item[StockSourceLinkInterface::STOCK_ID]][$item[SourceItemInterface::SKU]] = ++ $item[SourceItemInterface::SKU]; ++ } + +- $connection->query('SET group_concat_max_len = ' . $this->groupConcatMaxLen); +- $items = $connection->fetchAll($select); + return $this->getStockIdToSkuList($items); + } + +@@ -141,11 +138,11 @@ public function execute(array $sourceItemIds): array + private function getStockIdToSkuList(array $items): array + { + $skuListInStockList = []; +- foreach ($items as $item) { ++ foreach ($items as $stockId => $skuList) { + /** @var SkuListInStock $skuListInStock */ + $skuListInStock = $this->skuListInStockFactory->create(); +- $skuListInStock->setStockId((int)$item[StockSourceLinkInterface::STOCK_ID]); +- $skuListInStock->setSkuList(explode(',', $item[SourceItemInterface::SKU])); ++ $skuListInStock->setStockId((int)$stockId); ++ $skuListInStock->setSkuList($skuList); + $skuListInStockList[] = $skuListInStock; + } + return $skuListInStockList; +diff -Nuar a/vendor/magento/module-inventory-configurable-product-indexer/etc/di.xml b/vendor/magento/module-inventory-configurable-product-indexer/etc/di.xml +--- a/vendor/magento/module-inventory-configurable-product-indexer/etc/di.xml ++++ b/vendor/magento/module-inventory-configurable-product-indexer/etc/di.xml +@@ -27,7 +27,6 @@ + + + +- 2000 + Magento\Inventory\Model\ResourceModel\SourceItem::TABLE_NAME_SOURCE_ITEM + Magento\Inventory\Model\ResourceModel\StockSourceLink::TABLE_NAME_STOCK_SOURCE_LINK + diff --git a/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__grouped-product-indexer__1.0.3.patch b/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__grouped-product-indexer__1.0.3.patch new file mode 100644 index 0000000..9634753 --- /dev/null +++ b/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__grouped-product-indexer__1.0.3.patch @@ -0,0 +1,106 @@ +diff -Nuar a/vendor/magento/module-inventory-grouped-product-indexer/Indexer/SourceItem/SiblingSkuListInStockProvider.php b/vendor/magento/module-inventory-grouped-product-indexer/Indexer/SourceItem/SiblingSkuListInStockProvider.php +--- a/vendor/magento/module-inventory-grouped-product-indexer/Indexer/SourceItem/SiblingSkuListInStockProvider.php ++++ b/vendor/magento/module-inventory-grouped-product-indexer/Indexer/SourceItem/SiblingSkuListInStockProvider.php +@@ -32,10 +32,6 @@ class SiblingSkuListInStockProvider + */ + private $skuListInStockFactory; + +- /** +- * @var int +- */ +- private $groupConcatMaxLen; + /** + * @var MetadataPool + */ +@@ -52,12 +48,9 @@ class SiblingSkuListInStockProvider + private $tableNameStockSourceLink; + + /** +- * GetSkuListInStock constructor. +- * + * @param ResourceConnection $resourceConnection + * @param SkuListInStockFactory $skuListInStockFactory + * @param MetadataPool $metadataPool +- * @param int $groupConcatMaxLen + * @param string $tableNameSourceItem + * @param string $tableNameStockSourceLink + */ +@@ -65,13 +58,11 @@ public function __construct( + ResourceConnection $resourceConnection, + SkuListInStockFactory $skuListInStockFactory, + MetadataPool $metadataPool, +- int $groupConcatMaxLen, + $tableNameSourceItem, + $tableNameStockSourceLink + ) { + $this->resourceConnection = $resourceConnection; + $this->skuListInStockFactory = $skuListInStockFactory; +- $this->groupConcatMaxLen = $groupConcatMaxLen; + $this->metadataPool = $metadataPool; + $this->tableNameSourceItem = $tableNameSourceItem; + $this->tableNameStockSourceLink = $tableNameStockSourceLink; +@@ -92,15 +83,13 @@ public function execute(array $sourceItemIds): array + + $metadata = $this->metadataPool->getMetadata(ProductInterface::class); + $linkField = $metadata->getLinkField(); ++ $items = []; + + $select = $connection + ->select() + ->from( + ['source_item' => $sourceItemTable], +- [ +- SourceItemInterface::SKU => +- "GROUP_CONCAT(DISTINCT sibling_product_entity." . SourceItemInterface::SKU . " SEPARATOR ',')" +- ] ++ [SourceItemInterface::SKU => 'sibling_product_entity.' . SourceItemInterface::SKU] + )->joinInner( + ['stock_source_link' => $sourceStockLinkTable], + sprintf( +@@ -127,11 +116,16 @@ public function execute(array $sourceItemIds): array + ['sibling_product_entity' => $this->resourceConnection->getTableName('catalog_product_entity')], + 'sibling_product_entity.' . $linkField . ' = sibling_link.linked_product_id', + [] +- )->where('source_item.source_item_id IN (?)', $sourceItemIds) +- ->group(['stock_source_link.' . StockSourceLinkInterface::STOCK_ID]); ++ )->where( ++ 'source_item.source_item_id IN (?)', ++ $sourceItemIds ++ ); + +- $connection->query('SET group_concat_max_len = ' . $this->groupConcatMaxLen); +- $items = $connection->fetchAll($select); ++ $dbStatement = $connection->query($select); ++ while ($item = $dbStatement->fetch()) { ++ $items[$item[StockSourceLinkInterface::STOCK_ID]][$item[SourceItemInterface::SKU]] = ++ $item[SourceItemInterface::SKU]; ++ } + + return $this->getStockIdToSkuList($items); + } +@@ -145,11 +139,11 @@ public function execute(array $sourceItemIds): array + private function getStockIdToSkuList(array $items): array + { + $skuListInStockList = []; +- foreach ($items as $item) { ++ foreach ($items as $stockId => $skuList) { + /** @var SkuListInStock $skuListInStock */ + $skuListInStock = $this->skuListInStockFactory->create(); +- $skuListInStock->setStockId((int)$item[StockSourceLinkInterface::STOCK_ID]); +- $skuListInStock->setSkuList(explode(',', $item[SourceItemInterface::SKU])); ++ $skuListInStock->setStockId((int)$stockId); ++ $skuListInStock->setSkuList($skuList); + $skuListInStockList[] = $skuListInStock; + } + return $skuListInStockList; +diff -Nuar a/vendor/magento/module-inventory-grouped-product-indexer/etc/di.xml b/vendor/magento/module-inventory-grouped-product-indexer/etc/di.xml +--- a/vendor/magento/module-inventory-grouped-product-indexer/etc/di.xml ++++ b/vendor/magento/module-inventory-grouped-product-indexer/etc/di.xml +@@ -27,7 +27,6 @@ + + + +- 2000 + Magento\Inventory\Model\ResourceModel\SourceItem::TABLE_NAME_SOURCE_ITEM + Magento\Inventory\Model\ResourceModel\StockSourceLink::TABLE_NAME_STOCK_SOURCE_LINK + diff --git a/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__indexer__1.0.3.patch b/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__indexer__1.0.3.patch new file mode 100644 index 0000000..61a09e0 --- /dev/null +++ b/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__indexer__1.0.3.patch @@ -0,0 +1,99 @@ +diff -Nuar a/vendor/magento/module-inventory-indexer/Indexer/SourceItem/GetSkuListInStock.php b/vendor/magento/module-inventory-indexer/Indexer/SourceItem/GetSkuListInStock.php +--- a/vendor/magento/module-inventory-indexer/Indexer/SourceItem/GetSkuListInStock.php ++++ b/vendor/magento/module-inventory-indexer/Indexer/SourceItem/GetSkuListInStock.php +@@ -29,25 +29,15 @@ class GetSkuListInStock + private $skuListInStockFactory; + + /** +- * @var int +- */ +- private $groupConcatMaxLen; +- +- /** +- * GetSkuListInStock constructor. +- * + * @param ResourceConnection $resourceConnection + * @param SkuListInStockFactory $skuListInStockFactory +- * @param int $groupConcatMaxLen + */ + public function __construct( + ResourceConnection $resourceConnection, +- SkuListInStockFactory $skuListInStockFactory, +- int $groupConcatMaxLen ++ SkuListInStockFactory $skuListInStockFactory + ) { + $this->resourceConnection = $resourceConnection; + $this->skuListInStockFactory = $skuListInStockFactory; +- $this->groupConcatMaxLen = $groupConcatMaxLen; + } + + /** +@@ -65,15 +55,13 @@ public function execute(array $sourceItemIds): array + $sourceItemTable = $this->resourceConnection->getTableName( + SourceItemResourceModel::TABLE_NAME_SOURCE_ITEM + ); ++ $items = []; + + $select = $connection + ->select() + ->from( + ['source_item' => $sourceItemTable], +- [ +- SourceItemInterface::SKU => +- sprintf("GROUP_CONCAT(DISTINCT %s SEPARATOR ',')", 'source_item.' . SourceItemInterface::SKU) +- ] ++ [SourceItemInterface::SKU => 'source_item.' . SourceItemInterface::SKU] + )->joinInner( + ['stock_source_link' => $sourceStockLinkTable], + sprintf( +@@ -82,11 +70,16 @@ public function execute(array $sourceItemIds): array + StockSourceLink::SOURCE_CODE + ), + [StockSourceLink::STOCK_ID] +- )->where('source_item.source_item_id IN (?)', $sourceItemIds) +- ->group(['stock_source_link.' . StockSourceLink::STOCK_ID]); ++ )->where( ++ 'source_item.source_item_id IN (?)', ++ $sourceItemIds ++ ); ++ ++ $dbStatement = $connection->query($select); ++ while ($item = $dbStatement->fetch()) { ++ $items[$item[StockSourceLink::STOCK_ID]][$item[SourceItemInterface::SKU]] = $item[SourceItemInterface::SKU]; ++ } + +- $connection->query('SET group_concat_max_len = ' . $this->groupConcatMaxLen); +- $items = $connection->fetchAll($select); + return $this->getStockIdToSkuList($items); + } + +@@ -99,11 +92,11 @@ public function execute(array $sourceItemIds): array + private function getStockIdToSkuList(array $items): array + { + $skuListInStockList = []; +- foreach ($items as $item) { ++ foreach ($items as $stockId => $skuList) { + /** @var SkuListInStock $skuListInStock */ + $skuListInStock = $this->skuListInStockFactory->create(); +- $skuListInStock->setStockId((int)$item[StockSourceLink::STOCK_ID]); +- $skuListInStock->setSkuList(explode(',', $item[SourceItemInterface::SKU])); ++ $skuListInStock->setStockId((int)$stockId); ++ $skuListInStock->setSkuList($skuList); + $skuListInStockList[] = $skuListInStock; + } + return $skuListInStockList; +diff -Nuar a/vendor/magento/module-inventory-indexer/etc/di.xml b/vendor/magento/module-inventory-indexer/etc/di.xml +--- a/vendor/magento/module-inventory-indexer/etc/di.xml ++++ b/vendor/magento/module-inventory-indexer/etc/di.xml +@@ -25,11 +25,6 @@ + Magento\InventoryIndexer\Indexer\IndexHandler + + +- +- +- 2000 +- +- + + + catalog_product_entity diff --git a/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__reservations__1.0.3.patch b/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__reservations__1.0.3.patch new file mode 100644 index 0000000..96ad36d --- /dev/null +++ b/patches/MSI-GH-2515__eliminate_group_concat_from_source_item_indexer__reservations__1.0.3.patch @@ -0,0 +1,12 @@ +diff -Nuar a/vendor/magento/module-inventory-reservations/etc/di.xml b/vendor/magento/module-inventory-reservations/etc/di.xml +--- a/vendor/magento/module-inventory-reservations/etc/di.xml ++++ b/vendor/magento/module-inventory-reservations/etc/di.xml +@@ -13,7 +13,7 @@ + + + +- 2000 ++ 32768 + + + diff --git a/patches/SET-36__fix_oom_during_customer_import__2.1.11.patch b/patches/SET-36__fix_oom_during_customer_import__2.1.11.patch new file mode 100644 index 0000000..2d1d7e3 --- /dev/null +++ b/patches/SET-36__fix_oom_during_customer_import__2.1.11.patch @@ -0,0 +1,104 @@ +diff -Naur a/vendor/magento/module-customer-import-export/Model/Import/Address.php b/vendor/magento/module-customer-import-export/Model/Import/Address.php +index 3f745267fd2..13e659e4e62 100644 +--- a/vendor/magento/module-customer-import-export/Model/Import/Address.php ++++ b/vendor/magento/module-customer-import-export/Model/Import/Address.php +@@ -257,6 +257,11 @@ class Address extends AbstractCustomer + private $optionsByWebsite = []; + + /** ++ * @var array ++ */ ++ private $loadedAddresses; ++ ++ /** + * @param \Magento\Framework\Stdlib\StringUtils $string + * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig + * @param \Magento\ImportExport\Model\ImportFactory $importFactory +@@ -458,21 +463,50 @@ class Address extends AbstractCustomer + */ + protected function _initAddresses() + { +- /** @var $address \Magento\Customer\Model\Address */ +- foreach ($this->_addressCollection as $address) { +- $customerId = $address->getParentId(); +- if (!isset($this->_addresses[$customerId])) { +- $this->_addresses[$customerId] = []; ++ if ($this->_addressCollection->isLoaded()) { ++ /** @var $address \Magento\Customer\Model\Address */ ++ foreach ($this->_addressCollection as $address) { ++ $customerId = $address->getParentId(); ++ if (!isset($this->_addresses[$customerId])) { ++ $this->_addresses[$customerId] = []; ++ } ++ $addressId = $address->getId(); ++ if (!in_array($addressId, $this->_addresses[$customerId])) { ++ $this->_addresses[$customerId][] = $addressId; ++ } + } +- $addressId = $address->getId(); +- if (!in_array($addressId, $this->_addresses[$customerId])) { +- $this->_addresses[$customerId][] = $addressId; ++ } else { ++ foreach ($this->getLoadedAddresses() as $addressId => $address) { ++ $customerId = $address['parent_id']; ++ if (!isset($this->_addresses[$customerId])) { ++ $this->_addresses[$customerId] = []; ++ } ++ if (!in_array($addressId, $this->_addresses[$customerId])) { ++ $this->_addresses[$customerId][] = $addressId; ++ } + } + } + return $this; + } + + /** ++ * @return array ++ */ ++ private function getLoadedAddresses() ++ { ++ if (empty($this->loadedAddresses)) { ++ $collection = clone $this->_addressCollection; ++ $table = $collection->getMainTable(); ++ $select = $collection->getSelect(); ++ $select->reset('columns'); ++ $select->reset('from'); ++ $select->from($table, ['entity_id', 'parent_id']); ++ $this->loadedAddresses = $collection->getResource()->getConnection()->fetchAssoc($select); ++ } ++ return $this->loadedAddresses; ++ } ++ ++ /** + * Initialize country regions hash for clever recognition + * + * @return $this +diff -Naur a/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php b/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php +index 4e6687bff28..359822df6d9 100644 +--- a/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php ++++ b/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php +@@ -117,13 +117,18 @@ class Storage + */ + public function getCustomerId($email, $websiteId) + { +- // lazy loading +- $this->load(); ++ if (!isset($this->_customerIds[$email][$websiteId])) { ++ $collection = clone $this->_customerCollection; ++ $mainTable = $collection->getResource()->getEntityTable(); + +- if (isset($this->_customerIds[$email][$websiteId])) { +- return $this->_customerIds[$email][$websiteId]; +- } ++ $select = $collection->getSelect(); ++ $select->reset(); ++ $select->from($mainTable, ['entity_id']); ++ $select->where($mainTable . '.email = ?', $email); ++ $select->where($mainTable . '.website_id = ?', $websiteId); + +- return false; ++ $this->_customerIds[$email][$websiteId] = $collection->getResource()->getConnection()->fetchOne($select); ++ } ++ return $this->_customerIds[$email][$websiteId]; + } + } diff --git a/patches/SET-36__fix_oom_during_customer_import__2.1.4.patch b/patches/SET-36__fix_oom_during_customer_import__2.1.4.patch new file mode 100644 index 0000000..1e79c67 --- /dev/null +++ b/patches/SET-36__fix_oom_during_customer_import__2.1.4.patch @@ -0,0 +1,110 @@ +commit 4ee8443a262e18c08b942aef313710b2c070a7a4 +Author: Viktor Paladiichuk +Date: Thu Nov 16 18:55:15 2017 +0200 + + SET-36: Memory limit exhausted during import of customers and addresses + +diff -Naur a/vendor/magento/module-customer-import-export/Model/Import/Address.php b/vendor/magento/module-customer-import-export/Model/Import/Address.php +index eb5742d24c7..70b8c34ef41 100644 +--- a/vendor/magento/module-customer-import-export/Model/Import/Address.php ++++ b/vendor/magento/module-customer-import-export/Model/Import/Address.php +@@ -238,6 +238,11 @@ class Address extends AbstractCustomer + protected $postcodeValidator; + + /** ++ * @var array ++ */ ++ private $loadedAddresses; ++ ++ /** + * @param \Magento\Framework\Stdlib\StringUtils $string + * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig + * @param \Magento\ImportExport\Model\ImportFactory $importFactory +@@ -368,21 +373,50 @@ class Address extends AbstractCustomer + */ + protected function _initAddresses() + { +- /** @var $address \Magento\Customer\Model\Address */ +- foreach ($this->_addressCollection as $address) { +- $customerId = $address->getParentId(); +- if (!isset($this->_addresses[$customerId])) { +- $this->_addresses[$customerId] = []; ++ if ($this->_addressCollection->isLoaded()) { ++ /** @var $address \Magento\Customer\Model\Address */ ++ foreach ($this->_addressCollection as $address) { ++ $customerId = $address->getParentId(); ++ if (!isset($this->_addresses[$customerId])) { ++ $this->_addresses[$customerId] = []; ++ } ++ $addressId = $address->getId(); ++ if (!in_array($addressId, $this->_addresses[$customerId])) { ++ $this->_addresses[$customerId][] = $addressId; ++ } + } +- $addressId = $address->getId(); +- if (!in_array($addressId, $this->_addresses[$customerId])) { +- $this->_addresses[$customerId][] = $addressId; ++ } else { ++ foreach ($this->getLoadedAddresses() as $addressId => $address) { ++ $customerId = $address['parent_id']; ++ if (!isset($this->_addresses[$customerId])) { ++ $this->_addresses[$customerId] = []; ++ } ++ if (!in_array($addressId, $this->_addresses[$customerId])) { ++ $this->_addresses[$customerId][] = $addressId; ++ } + } + } + return $this; + } + + /** ++ * @return array ++ */ ++ private function getLoadedAddresses() ++ { ++ if (empty($this->loadedAddresses)) { ++ $collection = clone $this->_addressCollection; ++ $table = $collection->getMainTable(); ++ $select = $collection->getSelect(); ++ $select->reset('columns'); ++ $select->reset('from'); ++ $select->from($table, ['entity_id', 'parent_id']); ++ $this->loadedAddresses = $collection->getResource()->getConnection()->fetchAssoc($select); ++ } ++ return $this->loadedAddresses; ++ } ++ ++ /** + * Initialize country regions hash for clever recognition + * + * @return $this +diff -Naur a/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php b/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php +index 4e6687bff28..359822df6d9 100644 +--- a/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php ++++ b/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php +@@ -117,13 +117,18 @@ class Storage + */ + public function getCustomerId($email, $websiteId) + { +- // lazy loading +- $this->load(); ++ if (!isset($this->_customerIds[$email][$websiteId])) { ++ $collection = clone $this->_customerCollection; ++ $mainTable = $collection->getResource()->getEntityTable(); + +- if (isset($this->_customerIds[$email][$websiteId])) { +- return $this->_customerIds[$email][$websiteId]; +- } ++ $select = $collection->getSelect(); ++ $select->reset(); ++ $select->from($mainTable, ['entity_id']); ++ $select->where($mainTable . '.email = ?', $email); ++ $select->where($mainTable . '.website_id = ?', $websiteId); + +- return false; ++ $this->_customerIds[$email][$websiteId] = $collection->getResource()->getConnection()->fetchOne($select); ++ } ++ return $this->_customerIds[$email][$websiteId]; + } + } diff --git a/patches/SET-36__fix_oom_during_customer_import__2.2.0.patch b/patches/SET-36__fix_oom_during_customer_import__2.2.0.patch new file mode 100644 index 0000000..7f546b8 --- /dev/null +++ b/patches/SET-36__fix_oom_during_customer_import__2.2.0.patch @@ -0,0 +1,110 @@ +commit 4ee8443a262e18c08b942aef313710b2c070a7a4 +Author: Viktor Paladiichuk +Date: Thu Nov 16 18:55:15 2017 +0200 + + SET-36: Memory limit exhausted during import of customers and addresses + +diff -Nuar a/vendor/magento/module-customer-import-export/Model/Import/Address.php b/vendor/magento/module-customer-import-export/Model/Import/Address.php +index eb5742d24c7..70b8c34ef41 100644 +--- a/vendor/magento/module-customer-import-export/Model/Import/Address.php ++++ b/vendor/magento/module-customer-import-export/Model/Import/Address.php +@@ -238,6 +238,11 @@ class Address extends AbstractCustomer + protected $postcodeValidator; + + /** ++ * @var array ++ */ ++ private $loadedAddresses; ++ ++ /** + * @param \Magento\Framework\Stdlib\StringUtils $string + * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig + * @param \Magento\ImportExport\Model\ImportFactory $importFactory +@@ -368,21 +373,50 @@ class Address extends AbstractCustomer + */ + protected function _initAddresses() + { +- /** @var $address \Magento\Customer\Model\Address */ +- foreach ($this->_addressCollection as $address) { +- $customerId = $address->getParentId(); +- if (!isset($this->_addresses[$customerId])) { +- $this->_addresses[$customerId] = []; ++ if ($this->_addressCollection->isLoaded()) { ++ /** @var $address \Magento\Customer\Model\Address */ ++ foreach ($this->_addressCollection as $address) { ++ $customerId = $address->getParentId(); ++ if (!isset($this->_addresses[$customerId])) { ++ $this->_addresses[$customerId] = []; ++ } ++ $addressId = $address->getId(); ++ if (!in_array($addressId, $this->_addresses[$customerId])) { ++ $this->_addresses[$customerId][] = $addressId; ++ } + } +- $addressId = $address->getId(); +- if (!in_array($addressId, $this->_addresses[$customerId])) { +- $this->_addresses[$customerId][] = $addressId; ++ } else { ++ foreach ($this->getLoadedAddresses() as $addressId => $address) { ++ $customerId = $address['parent_id']; ++ if (!isset($this->_addresses[$customerId])) { ++ $this->_addresses[$customerId] = []; ++ } ++ if (!in_array($addressId, $this->_addresses[$customerId])) { ++ $this->_addresses[$customerId][] = $addressId; ++ } + } + } + return $this; + } + + /** ++ * @return array ++ */ ++ private function getLoadedAddresses() ++ { ++ if (empty($this->loadedAddresses)) { ++ $collection = clone $this->_addressCollection; ++ $table = $collection->getMainTable(); ++ $select = $collection->getSelect(); ++ $select->reset('columns'); ++ $select->reset('from'); ++ $select->from($table, ['entity_id', 'parent_id']); ++ $this->loadedAddresses = $collection->getResource()->getConnection()->fetchAssoc($select); ++ } ++ return $this->loadedAddresses; ++ } ++ ++ /** + * Initialize country regions hash for clever recognition + * + * @return $this +diff -Nuar a/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php b/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php +index 4e6687bff28..359822df6d9 100644 +--- a/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php ++++ b/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php +@@ -117,13 +117,18 @@ class Storage + */ + public function getCustomerId($email, $websiteId) + { +- // lazy loading +- $this->load(); ++ if (!isset($this->_customerIds[$email][$websiteId])) { ++ $collection = clone $this->_customerCollection; ++ $mainTable = $collection->getResource()->getEntityTable(); + +- if (isset($this->_customerIds[$email][$websiteId])) { +- return $this->_customerIds[$email][$websiteId]; +- } ++ $select = $collection->getSelect(); ++ $select->reset(); ++ $select->from($mainTable, ['entity_id']); ++ $select->where($mainTable . '.email = ?', $email); ++ $select->where($mainTable . '.website_id = ?', $websiteId); + +- return false; ++ $this->_customerIds[$email][$websiteId] = $collection->getResource()->getConnection()->fetchOne($select); ++ } ++ return $this->_customerIds[$email][$websiteId]; + } + } diff --git a/patches/SET-36__fix_oom_during_customer_import__2.2.4.patch b/patches/SET-36__fix_oom_during_customer_import__2.2.4.patch new file mode 100644 index 0000000..6db2e0e --- /dev/null +++ b/patches/SET-36__fix_oom_during_customer_import__2.2.4.patch @@ -0,0 +1,102 @@ +diff -Nuar a/vendor/magento/module-customer-import-export/Model/Import/Address.php b/vendor/magento/module-import-export/Model/Import/Address.php +index 1e1221e..41e0512 100644 +--- a/vendor/magento/module-customer-import-export/Model/Import/Address.php ++++ b/vendor/magento/module-customer-import-export/Model/Import/Address.php +@@ -253,6 +253,11 @@ class Address extends AbstractCustomer + private $optionsByWebsite = []; + + /** ++ * @var array ++ */ ++ private $loadedAddresses; ++ ++ /** + * @param \Magento\Framework\Stdlib\StringUtils $string + * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig + * @param \Magento\ImportExport\Model\ImportFactory $importFactory +@@ -443,20 +448,49 @@ class Address extends AbstractCustomer + */ + protected function _initAddresses() + { +- /** @var $address \Magento\Customer\Model\Address */ +- foreach ($this->_addressCollection as $address) { +- $customerId = $address->getParentId(); +- if (!isset($this->_addresses[$customerId])) { +- $this->_addresses[$customerId] = []; ++ if ($this->_addressCollection->isLoaded()) { ++ /** @var $address \Magento\Customer\Model\Address */ ++ foreach ($this->_addressCollection as $address) { ++ $customerId = $address->getParentId(); ++ if (!isset($this->_addresses[$customerId])) { ++ $this->_addresses[$customerId] = []; ++ } ++ $addressId = $address->getId(); ++ if (!in_array($addressId, $this->_addresses[$customerId])) { ++ $this->_addresses[$customerId][] = $addressId; ++ } + } +- $addressId = $address->getId(); +- if (!in_array($addressId, $this->_addresses[$customerId])) { +- $this->_addresses[$customerId][] = $addressId; ++ } else { ++ foreach ($this->getLoadedAddresses() as $addressId => $address) { ++ $customerId = $address['parent_id']; ++ if (!isset($this->_addresses[$customerId])) { ++ $this->_addresses[$customerId] = []; ++ } ++ if (!in_array($addressId, $this->_addresses[$customerId])) { ++ $this->_addresses[$customerId][] = $addressId; ++ } + } + } + return $this; + } + ++ /** ++ * @return array ++ */ ++ private function getLoadedAddresses() ++ { ++ if (empty($this->loadedAddresses)) { ++ $collection = clone $this->_addressCollection; ++ $table = $collection->getMainTable(); ++ $select = $collection->getSelect(); ++ $select->reset('columns'); ++ $select->reset('from'); ++ $select->from($table, ['entity_id', 'parent_id']); ++ $this->loadedAddresses = $collection->getResource()->getConnection()->fetchAssoc($select); ++ } ++ return $this->loadedAddresses; ++ } ++ + /** + * Initialize country regions hash for clever recognition + * +diff -Nuar a/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php b/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php +index ae88e96..a6f7aa3 100644 +--- a/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php ++++ b/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php +@@ -117,13 +117,18 @@ class Storage + */ + public function getCustomerId($email, $websiteId) + { +- // lazy loading +- $this->load(); ++ if (!isset($this->_customerIds[$email][$websiteId])) { ++ $collection = clone $this->_customerCollection; ++ $mainTable = $collection->getResource()->getEntityTable(); + +- if (isset($this->_customerIds[$email][$websiteId])) { +- return $this->_customerIds[$email][$websiteId]; ++ $select = $collection->getSelect(); ++ $select->reset(); ++ $select->from($mainTable, ['entity_id']); ++ $select->where($mainTable . '.email = ?', $email); ++ $select->where($mainTable . '.website_id = ?', $websiteId); ++ $this->_customerIds[$email][$websiteId] = $collection->getResource()->getConnection()->fetchOne($select); + } + +- return false; ++ return $this->_customerIds[$email][$websiteId]; + } + } diff --git a/src/App/Container.php b/src/App/Container.php new file mode 100644 index 0000000..3620b92 --- /dev/null +++ b/src/App/Container.php @@ -0,0 +1,96 @@ +set('container', $containerBuilder); + $containerBuilder->setDefinition('container', new Definition(__CLASS__)) + ->setArguments([$basePath, $magentoBasePath]); + + $directoryList = new DirectoryList($basePath, $magentoBasePath); + $composer = $this->createComposerInstance($directoryList); + + $containerBuilder->set(DirectoryList::class, $directoryList); + $containerBuilder->setDefinition(DirectoryList::class, new Definition(DirectoryList::class)); + + $containerBuilder->set(Composer\Composer::class, $composer); + $containerBuilder->setDefinition(Composer\Composer::class, new Definition(Composer\Composer::class)); + + $loader = new XmlFileLoader($containerBuilder, new FileLocator($basePath . '/config')); + $loader->load('services.xml'); + $containerBuilder->compile(); + + $this->container = $containerBuilder; + } + + /** + * @param DirectoryList $directoryList + * @return Composer\Composer + */ + private function createComposerInstance(DirectoryList $directoryList): Composer\Composer + { + $composerFactory = new Composer\Factory(); + $composerFile = file_exists($directoryList->getMagentoRoot() . '/composer.json') + ? $directoryList->getMagentoRoot() . '/composer.json' + : $directoryList->getRoot() . '/composer.json'; + + $composer = $composerFactory->createComposer( + new Composer\IO\BufferIO(), + $composerFile, + false, + $directoryList->getMagentoRoot() + ); + + return $composer; + } + + /** + * @inheritDoc + */ + public function get($id) + { + return $this->container->get($id); + } + + /** + * @inheritdoc + */ + public function has($id): bool + { + return $this->container->has($id); + } +} diff --git a/src/App/GenericException.php b/src/App/GenericException.php new file mode 100644 index 0000000..06955e8 --- /dev/null +++ b/src/App/GenericException.php @@ -0,0 +1,26 @@ +container = $container; + + parent::__construct( + $container->get(Composer::class)->getPackage()->getPrettyName(), + $container->get(Composer::class)->getPackage()->getPrettyVersion() + ); + } + + /** + * @inheritdoc + */ + protected function getDefaultCommands() + { + return array_merge(parent::getDefaultCommands(), [ + $this->container->get(Command\Apply::class), + ]); + } +} diff --git a/src/Command/Apply.php b/src/Command/Apply.php new file mode 100644 index 0000000..63f9659 --- /dev/null +++ b/src/Command/Apply.php @@ -0,0 +1,72 @@ +manager = $manager; + + parent::__construct(self::NAME); + } + + /** + * @inheritDoc + */ + protected function configure() + { + $this->setName(self::NAME) + ->setDescription('Apply patches') + ->addOption( + self::OPT_GIT_INSTALLATION, + null, + InputOption::VALUE_OPTIONAL, + 'Is git installation', + false + ); + + parent::configure(); + } + + /** + * {@inheritDoc} + * + * @throws ManagerException + * @throws ApplierException + */ + public function execute(InputInterface $input, OutputInterface $output) + { + $this->manager->copyStaticFile($output); + $this->manager->applyComposerPatches($input, $output); + $this->manager->applyHotFixes($input, $output); + } +} diff --git a/src/Command/Patch/Manager.php b/src/Command/Patch/Manager.php new file mode 100644 index 0000000..9fd0f0e --- /dev/null +++ b/src/Command/Patch/Manager.php @@ -0,0 +1,185 @@ +applier = $applier; + $this->filesystem = $filesystem; + $this->fileList = $fileList; + $this->directoryList = $directoryList; + } + + /** + * Copying static file endpoint. + * This resolves issue MAGECLOUD-314. + * + * @param OutputInterface $output + */ + public function copyStaticFile(OutputInterface $output) + { + $magentoRoot = $this->directoryList->getMagentoRoot(); + + if (!$this->filesystem->exists($magentoRoot . '/pub/static.php')) { + $output->writeln('File "static.php" was not found'); + + return; + } + + $this->filesystem->copy($magentoRoot . '/pub/static.php', $magentoRoot . '/pub/front-static.php'); + + $output->writeln('File "static.php" was copied'); + } + + /** + * Applies patches from composer.json file. + * Patches are applying from top to bottom of config list. + * + * ``` + * "colinmollenhour/credis" : { + * "Fix Redis issue": { + * "1.6": "patches/redis-pipeline.patch" + * } + * } + * + * Each patch must have corresponding constraint of target package, + * in one of the following format: + * - 1.6 + * - 1.6.* + * - ^1.6 + * + * @param InputInterface $input + * @param OutputInterface $output + * + * @throws ManagerException + * @throws ApplierException + */ + public function applyComposerPatches(InputInterface $input, OutputInterface $output) + { + try { + $patches = json_decode( + $this->filesystem->get($this->fileList->getPatches()) ?? '', + true + ); + } catch (FileNotFoundException $exception) { + throw new ManagerException($exception->getMessage(), $exception->getCode(), $exception); + } + + if (!$patches) { + $output->writeln('Composer patches not found'); + + return; + } + + $deployedFromGit = $input->getOption(Apply::OPT_GIT_INSTALLATION); + + foreach ($patches as $packageName => $patchesInfo) { + foreach ($patchesInfo as $patchName => $packageInfo) { + if (!is_array($packageInfo)) { + throw new ManagerException('Wrong patch constraints'); + } + + foreach ($packageInfo as $constraint => $path) { + $message = $this->applier->apply( + (string)$path, + (string)$patchName, + (string)$packageName, + (string)$constraint, + $deployedFromGit + ); + + if (null !== $message) { + $output->writeln($message); + } + } + } + } + } + + /** + * Applies patches from root directory m2-hotfixes. + * + * @param InputInterface $input + * @param OutputInterface $output + * + * @throws ApplierException + */ + public function applyHotFixes(InputInterface $input, OutputInterface $output) + { + $hotFixesDir = $this->directoryList->getMagentoRoot() . '/' . static::HOT_FIXES_DIR; + + if (!$this->filesystem->isDirectory($hotFixesDir)) { + $output->writeln('Hot-fixes directory was not found. Skipping'); + + return; + } + + $files = glob($hotFixesDir . '/*.patch'); + sort($files); + + $deployedFromGit = $input->getOption(Apply::OPT_GIT_INSTALLATION); + + $output->writeln('Applying hot-fixes'); + + foreach ($files as $file) { + $output->writeln( + $this->applier->applyFile($file, $deployedFromGit) + ); + } + } +} diff --git a/src/Command/Patch/ManagerException.php b/src/Command/Patch/ManagerException.php new file mode 100644 index 0000000..81aaae6 --- /dev/null +++ b/src/Command/Patch/ManagerException.php @@ -0,0 +1,17 @@ +root = realpath($root); + $this->magentoRoot = realpath($magentoRoot); + } + + /** + * @return string + */ + public function getRoot(): string + { + return $this->root; + } + + /** + * @return string + */ + public function getMagentoRoot(): string + { + return $this->magentoRoot; + } + + /** + * @return string + */ + public function getPatches(): string + { + return $this->getRoot() . '/patches'; + } +} diff --git a/src/Filesystem/FileList.php b/src/Filesystem/FileList.php new file mode 100644 index 0000000..10fd856 --- /dev/null +++ b/src/Filesystem/FileList.php @@ -0,0 +1,35 @@ +directoryList = $directoryList; + } + + /** + * @return string + */ + public function getPatches(): string + { + return $this->directoryList->getRoot() . '/patches.json'; + } +} diff --git a/src/Patch/Applier.php b/src/Patch/Applier.php new file mode 100644 index 0000000..d771bde --- /dev/null +++ b/src/Patch/Applier.php @@ -0,0 +1,172 @@ +repository = $composer->getRepositoryManager()->getLocalRepository(); + $this->processFactory = $processFactory; + $this->directoryList = $directoryList; + $this->filesystem = $file; + } + + /** + * @param string $path + * @param bool $deployedFromGit + * @return string + * + * @throws ApplierException + */ + public function applyFile(string $path, bool $deployedFromGit): string + { + return $this->processApply($path, $path, $deployedFromGit); + } + + /** + * Applies patch, using 'git apply' command. + * + * If the patch fails to apply, checks if it has already been applied which is considered ok. + * + * @param string $path Path to patch + * @param string $name Name of patch + * @param string $packageName Name of package to be patched + * @param string $constraint Specific constraint of package to be fixed + * @param bool $deployedFromGit + * @return string|null + * + * @throws ApplierException + */ + public function apply( + string $path, + string $name, + string $packageName, + string $constraint, + bool $deployedFromGit + ) { + $fullName = sprintf( + '%s %s', + sprintf('%s (%s)', $name, $path), + $constraint + ); + + if ($packageName && !$this->matchConstraint($packageName, $constraint)) { + return null; + } + + /** + * Support for relative paths. + */ + if (!$this->filesystem->exists($path)) { + $path = $this->directoryList->getPatches() . '/' . $path; + } + + return $this->processApply($path, $fullName, $deployedFromGit); + } + + /** + * General apply processing. + * + * @param string $path + * @param string $fullName + * @param bool $deployedFromGit + * @return string + * + * @throws ApplierException + */ + private function processApply(string $path, string $fullName, bool $deployedFromGit): string + { + try { + $this->processFactory->create(['git', 'apply', $path]) + ->disableOutput() + ->mustRun(); + } catch (ProcessFailedException $exception) { + if ($deployedFromGit) { + return sprintf( + 'Patch "%s" was not applied. (%s)', + $fullName, + $exception->getMessage() + ); + } + + try { + $this->processFactory->create(['git', 'apply', '--check', '--reverse', $path]) + ->disableOutput() + ->mustRun(); + } catch (ProcessFailedException $reverseException) { + throw new ApplierException( + $reverseException->getMessage(), + $reverseException->getCode(), + $reverseException + ); + } + + return sprintf( + 'Patch "%s" was already applied', + $fullName + ); + } + + return sprintf( + 'Patch "%s" applied', + $fullName + ); + } + + /** + * Checks whether package with specific constraint exists in the system. + * + * @param string $packageName + * @param string $constraint + * @return bool True if patch with provided constraint exists, false otherwise. + */ + private function matchConstraint(string $packageName, string $constraint): bool + { + return $this->repository->findPackage($packageName, $constraint) instanceof Composer\Package\PackageInterface; + } +} diff --git a/src/Patch/ApplierException.php b/src/Patch/ApplierException.php new file mode 100644 index 0000000..2058a0d --- /dev/null +++ b/src/Patch/ApplierException.php @@ -0,0 +1,17 @@ +directoryList = $directoryList; + } + + /** + * @param array $cmd + * @return Process + */ + public function create(array $cmd): Process + { + return new Process($cmd, $this->directoryList->getMagentoRoot()); + } +} diff --git a/src/Test/Unit/Command/ApplyTest.php b/src/Test/Unit/Command/ApplyTest.php new file mode 100644 index 0000000..4135571 --- /dev/null +++ b/src/Test/Unit/Command/ApplyTest.php @@ -0,0 +1,64 @@ +managerMock = $this->createMock(Manager::class); + + $this->command = new Apply( + $this->managerMock + ); + } + + /** + * @throws FileNotFoundException + */ + public function testExecute() + { + /** @var InputInterface|MockObject $inputMock */ + $inputMock = $this->getMockForAbstractClass(InputInterface::class); + /** @var OutputInterface|MockObject $outputMock */ + $outputMock = $this->getMockForAbstractClass(OutputInterface::class); + + $this->managerMock->expects($this->once()) + ->method('copyStaticFile'); + $this->managerMock->expects($this->once()) + ->method('applyComposerPatches'); + $this->managerMock->expects($this->once()) + ->method('applyHotFixes'); + + $this->command->execute($inputMock, $outputMock); + } +} diff --git a/src/Test/Unit/Command/Patch/ManagerTest.php b/src/Test/Unit/Command/Patch/ManagerTest.php new file mode 100644 index 0000000..0342031 --- /dev/null +++ b/src/Test/Unit/Command/Patch/ManagerTest.php @@ -0,0 +1,229 @@ +applierMock = $this->createMock(Applier::class); + $this->composerPackageMock = $this->getMockForAbstractClass(RootPackageInterface::class); + $this->filesystemMock = $this->createMock(Filesystem::class); + $this->directoryListMock = $this->createMock(DirectoryList::class); + $this->fileListMock = $this->createMock(FileList::class); + + $this->manager = new Manager( + $this->applierMock, + $this->filesystemMock, + $this->fileListMock, + $this->directoryListMock + ); + } + + public function testCopyStaticFiles() + { + $this->filesystemMock->expects($this->once()) + ->method('exists') + ->with('/pub/static.php') + ->willReturn(true); + $this->filesystemMock->expects($this->once()) + ->method('copy') + ->with('/pub/static.php', '/pub/front-static.php') + ->willReturn(true); + + /** @var OutputInterface|MockObject $outputMock */ + $outputMock = $this->getMockForAbstractClass(OutputInterface::class); + $outputMock->expects($this->once()) + ->method('writeln') + ->with('File "static.php" was copied'); + + $this->manager->copyStaticFile($outputMock); + } + + /** + * @throws ApplierException + * @throws ManagerException + */ + public function testApplyComposerPatches() + { + $this->filesystemMock->expects($this->once()) + ->method('get') + ->willReturn(json_encode( + [ + 'package1' => [ + 'patchName1' => [ + '100' => 'patchPath1', + ], + ], + 'package2' => [ + 'patchName2' => [ + '101.*' => 'patchPath2', + ], + 'patchName3' => [ + '102.*' => 'patchPath3', + ], + ] + ] + )); + $this->applierMock->expects($this->exactly(3)) + ->method('apply') + ->withConsecutive( + ['patchPath1', 'patchName1', 'package1', '100'], + ['patchPath2', 'patchName2', 'package2', '101.*'], + ['patchPath3', 'patchName3', 'package2', '102.*'] + ); + + /** @var InputInterface|MockObject $inputMock */ + $inputMock = $this->getMockForAbstractClass(InputInterface::class); + /** @var OutputInterface|MockObject $outputMock */ + $outputMock = $this->getMockForAbstractClass(OutputInterface::class); + + $inputMock->method('getOption') + ->with(Apply::OPT_GIT_INSTALLATION) + ->willReturn(false); + + $this->manager->applyComposerPatches($inputMock, $outputMock); + } + + /** + * @expectedException \Magento\CloudPatches\Command\Patch\ManagerException + * @expectedExceptionMessage Not Found + * + * @throws ApplierException + * @throws ManagerException + */ + public function testApplyComposerPatchesWithFSException() + { + $this->filesystemMock->expects($this->once()) + ->method('get') + ->willThrowException(new FileNotFoundException('Not Found')); + + /** @var InputInterface|MockObject $inputMock */ + $inputMock = $this->getMockForAbstractClass(InputInterface::class); + /** @var OutputInterface|MockObject $outputMock */ + $outputMock = $this->getMockForAbstractClass(OutputInterface::class); + + $inputMock->method('getOption') + ->with(Apply::OPT_GIT_INSTALLATION) + ->willReturn(false); + + $this->manager->applyComposerPatches($inputMock, $outputMock); + } + + /** + * @throws ApplierException + */ + public function testExecuteApplyHotFixes() + { + $this->directoryListMock->expects($this->any()) + ->method('getMagentoRoot') + ->willReturn(__DIR__ . '/_files'); + $this->filesystemMock->expects($this->once()) + ->method('isDirectory') + ->willReturn(true); + $this->applierMock->expects($this->exactly(2)) + ->method('applyFile') + ->willReturnMap([ + [__DIR__ . '/_files/' . Manager::HOT_FIXES_DIR . '/patch1.patch', false, 'Patch 1 applied'], + [__DIR__ . '/_files/' . Manager::HOT_FIXES_DIR . '/patch2.patch', false, 'Patch 2 applied'] + ]); + + /** @var InputInterface|MockObject $inputMock */ + $inputMock = $this->getMockForAbstractClass(InputInterface::class); + $inputMock->expects($this->once()) + ->method('getOption') + ->with(Apply::OPT_GIT_INSTALLATION) + ->willReturn(false); + /** @var OutputInterface|MockObject $outputMock */ + $outputMock = $this->getMockForAbstractClass(OutputInterface::class); + $outputMock->expects($this->exactly(3)) + ->method('writeln') + ->withConsecutive( + ['Applying hot-fixes', 0], + ['Patch 1 applied', 0], + ['Patch 2 applied', 0] + ); + + $this->manager->applyHotFixes($inputMock, $outputMock); + } + + /** + * @throws ApplierException + */ + public function testExecuteApplyHotFixesNotFound() + { + $this->directoryListMock->expects($this->any()) + ->method('getMagentoRoot') + ->willReturn(__DIR__ . '/_files'); + $this->filesystemMock->expects($this->once()) + ->method('isDirectory') + ->willReturn(false); + /** @var InputInterface|MockObject $inputMock */ + $inputMock = $this->getMockForAbstractClass(InputInterface::class); + /** @var OutputInterface|MockObject $outputMock */ + $outputMock = $this->getMockForAbstractClass(OutputInterface::class); + $outputMock->expects($this->once()) + ->method('writeln') + ->withConsecutive( + ['Hot-fixes directory was not found. Skipping', 0] + ); + + $this->manager->applyHotFixes($inputMock, $outputMock); + } +} diff --git a/src/Test/Unit/Command/Patch/_files/m2-hotfixes/patch1.patch b/src/Test/Unit/Command/Patch/_files/m2-hotfixes/patch1.patch new file mode 100644 index 0000000..e69de29 diff --git a/src/Test/Unit/Command/Patch/_files/m2-hotfixes/patch2.patch b/src/Test/Unit/Command/Patch/_files/m2-hotfixes/patch2.patch new file mode 100644 index 0000000..e69de29 diff --git a/src/Test/Unit/Command/Patch/_files/m2-hotfixes/readme.md b/src/Test/Unit/Command/Patch/_files/m2-hotfixes/readme.md new file mode 100644 index 0000000..e69de29 diff --git a/src/Test/Unit/Filesystem/DirectoryListTest.php b/src/Test/Unit/Filesystem/DirectoryListTest.php new file mode 100644 index 0000000..c2ee383 --- /dev/null +++ b/src/Test/Unit/Filesystem/DirectoryListTest.php @@ -0,0 +1,67 @@ +directoryList = new DirectoryList( + $this->root, + $this->magentoRoot + ); + } + + public function testGetRoot() + { + $this->assertSame( + $this->root, + $this->directoryList->getRoot() + ); + } + + public function testGetMagentoRoot() + { + $this->assertSame( + $this->magentoRoot, + $this->directoryList->getMagentoRoot() + ); + } + + public function testGetPatches() + { + $this->assertSame( + $this->root . '/patches', + $this->directoryList->getPatches() + ); + } +} diff --git a/src/Test/Unit/Filesystem/FileListTest.php b/src/Test/Unit/Filesystem/FileListTest.php new file mode 100644 index 0000000..5d3f8b9 --- /dev/null +++ b/src/Test/Unit/Filesystem/FileListTest.php @@ -0,0 +1,51 @@ +directoryListMock = $this->createMock(DirectoryList::class); + + $this->directoryListMock->method('getRoot') + ->willReturn('root'); + + $this->fileList = new FileList( + $this->directoryListMock + ); + } + + public function testGetPatches() + { + $this->assertSame( + 'root/patches.json', + $this->fileList->getPatches() + ); + } +} diff --git a/src/Test/Unit/Filesystem/_files/.gitignore b/src/Test/Unit/Filesystem/_files/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/src/Test/Unit/Patch/ApplierTest.php b/src/Test/Unit/Patch/ApplierTest.php new file mode 100644 index 0000000..ac488dd --- /dev/null +++ b/src/Test/Unit/Patch/ApplierTest.php @@ -0,0 +1,298 @@ +composerMock = $this->createMock(Composer::class); + $this->localRepositoryMock = $this->getMockForAbstractClass(RepositoryInterface::class); + $this->directoryListMock = $this->createMock(DirectoryList::class); + $this->filesystemMock = $this->createMock(Filesystem::class); + $this->processFactoryMock = $this->createMock(ProcessFactory::class); + + $repositoryManagerMock = $this->createMock(RepositoryManager::class); + + $repositoryManagerMock->expects($this->once()) + ->method('getLocalRepository') + ->willReturn($this->localRepositoryMock); + $this->composerMock->expects($this->once()) + ->method('getRepositoryManager') + ->willReturn($repositoryManagerMock); + + $this->applier = new Applier( + $this->composerMock, + $this->processFactoryMock, + $this->directoryListMock, + $this->filesystemMock + ); + } + + /** + * @param string $path + * @param string $name + * @param string $packageName + * @param string $constraint + * @param string $expectedLog + * @dataProvider applyDataProvider + * + * @throws ApplierException + */ + public function testApply(string $path, string $name, string $packageName, string $constraint, string $expectedLog) + { + $this->filesystemMock->expects($this->once()) + ->method('exists') + ->with($path) + ->willReturn(true); + $this->localRepositoryMock->method('findPackage') + ->with($packageName, $constraint) + ->willReturn($this->getMockForAbstractClass(PackageInterface::class)); + + $processMock = $this->createMock(Process::class); + + $this->processFactoryMock->expects($this->once()) + ->method('create') + ->with(['git', 'apply', $path]) + ->willReturn($processMock); + $processMock->expects($this->once()) + ->method('disableOutput') + ->willReturnSelf(); + $processMock->expects($this->once()) + ->method('mustRun'); + + $this->assertSame( + $expectedLog, + $this->applier->apply($path, $name, $packageName, $constraint, false) + ); + } + + /** + * @return array + */ + public function applyDataProvider(): array + { + return [ + ['path/to/patch', 'patchName', 'packageName', '1.0', 'Patch "patchName (path/to/patch) 1.0" applied'] + ]; + } + + /** + * @param string $path + * @param string $expectedLog + * @dataProvider applyFileDataProvider + * + * @throws ApplierException + */ + public function testApplyFile(string $path, string $expectedLog) + { + $processMock = $this->createMock(Process::class); + + $this->processFactoryMock->expects($this->once()) + ->method('create') + ->with(['git', 'apply', $path]) + ->willReturn($processMock); + $processMock->expects($this->once()) + ->method('disableOutput') + ->willReturnSelf(); + $processMock->expects($this->once()) + ->method('mustRun'); + + $this->assertSame( + $expectedLog, + $this->applier->applyFile($path, false) + ); + } + + /** + * @return array + */ + public function applyFileDataProvider(): array + { + return [ + ['path/to/patch2', 'Patch "path/to/patch2" applied'], + ]; + } + + /** + * @throws ApplierException + */ + public function testApplyPathNotExists() + { + $path = 'path/to/patch'; + $name = 'patchName'; + $packageName = 'packageName'; + $constraint = '1.0'; + + $this->filesystemMock->expects($this->once()) + ->method('exists') + ->with($path) + ->willReturn(false); + $this->localRepositoryMock->expects($this->once()) + ->method('findPackage') + ->with($packageName, $constraint) + ->willReturn($this->getMockForAbstractClass(PackageInterface::class)); + $this->directoryListMock->expects($this->once()) + ->method('getPatches') + ->willReturn('root'); + + $processMock = $this->createMock(Process::class); + + $this->processFactoryMock->expects($this->once()) + ->method('create') + ->with(['git', 'apply', 'root/path/to/patch']) + ->willReturn($processMock); + $processMock->expects($this->once()) + ->method('disableOutput') + ->willReturnSelf(); + $processMock->expects($this->once()) + ->method('mustRun'); + + $this->applier->apply($path, $name, $packageName, $constraint, false); + } + + /** + * @throws ApplierException + */ + public function testApplyPathNotExistsAndNotMatchedConstraints() + { + $path = 'path/to/patch'; + $name = 'patchName'; + $packageName = 'packageName'; + $constraint = '1.0'; + + $this->localRepositoryMock->expects($this->once()) + ->method('findPackage') + ->with($packageName, $constraint) + ->willReturn(null); + + $processMock = $this->createMock(Process::class); + + $this->processFactoryMock->expects($this->never()) + ->method('create') + ->with(['git', 'apply', 'root/path/to/patch']) + ->willReturn($processMock); + + $this->applier->apply($path, $name, $packageName, $constraint, false); + } + + /** + * @throws ApplierException + */ + public function testApplyPatchAlreadyApplied() + { + $path = 'path/to/patch'; + $name = 'patchName'; + $packageName = 'packageName'; + $constraint = '1.0'; + + $this->filesystemMock->expects($this->once()) + ->method('exists') + ->with($path) + ->willReturn(true); + $this->localRepositoryMock->expects($this->once()) + ->method('findPackage') + ->with($packageName, $constraint) + ->willReturn($this->getMockForAbstractClass(PackageInterface::class)); + + $this->processFactoryMock->expects($this->exactly(2)) + ->method('create') + ->willReturnMap([ + [['git', 'apply', 'path/to/patch']], + [['git', 'apply', 'path/to/patch', '--revert']] + ])->willReturnCallback([$this, 'shellMockReverseCallback']); + + $this->assertSame( + 'Patch "patchName (path/to/patch) 1.0" was already applied', + $this->applier->apply($path, $name, $packageName, $constraint, false) + ); + } + + /** + * @param array $command + * @return Process + * + * @throws ProcessFailedException when the command isn't a reverse + */ + public function shellMockReverseCallback(array $command): Process + { + if (in_array('--reverse', $command, true) && in_array('--check', $command, true)) { + // Command was the reverse check, it's all good. + /** @var Process|MockObject $result */ + $result = $this->createMock(Process::class); + $result->expects($this->once()) + ->method('disableOutput') + ->willReturnSelf(); + $result->expects($this->once()) + ->method('mustRun'); + + return $result; + } + + /** @var Process|MockObject $result */ + $result = $this->createMock(Process::class); + $result->expects($this->once()) + ->method('disableOutput') + ->willReturnSelf(); + $result->expects($this->once()) + ->method('mustRun') + ->willThrowException(new ProcessFailedException($result)); + + return $result; + } +} diff --git a/tests/static/Sniffs/Directives/StrictTypesSniff.php b/tests/static/Sniffs/Directives/StrictTypesSniff.php new file mode 100644 index 0000000..9bd9728 --- /dev/null +++ b/tests/static/Sniffs/Directives/StrictTypesSniff.php @@ -0,0 +1,120 @@ +getTokens(); + + // Tokens to look for. + $findTokens = [ + T_DECLARE, + T_NAMESPACE, + T_CLASS + ]; + + // Find the first occurrence of the tokens to look for. + $position = $phpcsFile->findNext($findTokens, $stackPtr); + + // If the first token found is not T_DECLARE, then the file does not include a strict_types declaration. + if($tokens[$position]['code'] !== T_DECLARE) { + // Fix and set the boolean flag to true. + $this->fix($phpcsFile, $position); + } + + // If the file includes a declare directive, and the file has not already been fixed, scan specifically + // for strict_types and fix as needed. + if(!$this->fixed) { + if(!$this->scan($phpcsFile, $tokens, $position)) { + $this->fix($phpcsFile, $position); + } + } + } + + /** + * Fixer to add the strict_types declaration. + * + * @param File $phpcsFile + * @param int $position + */ + private function fix(File $phpcsFile, int $position) : void + { + // Get the fixer. + $fixer = $phpcsFile->fixer; + // Record the error. + $phpcsFile->addFixableError("Missing strict_types declaration", $position, self::class); + // Prepend content at the given position. + $fixer->addContentBefore($position, "declare(strict_types=1);\n\n"); + // Set flag. + $this->fixed = true; + } + + /** + * Recursive method to scan declare statements for strict_types. + * + * @param File $phpcsFile + * @param array $tokens + * @param int $position + * @return bool + */ + private function scan(File $phpcsFile, array $tokens, int $position) : bool + { + // Exit statement, if the beginning of the file has been reached. + if($tokens[$position]['code'] === T_OPEN_TAG || $position === 0) { + return false; + } + + if(!$phpcsFile->findNext([T_STRING], $position)) { + // If there isn't a T_STRING token for the declare directive, continue scan. + return $this->scan($phpcsFile, $tokens, $phpcsFile->findPrevious([T_DECLARE], $position - 1)); + } else { + // Checking specifically for strict_types. + $temp = $phpcsFile->findNext([T_STRING], $position); + if($tokens[$temp]['content'] === 'strict_types') { + // Return true as strict_types directive has been found. + return true; + } else { + // Continue scan if strict_types hasn't been found. + return $this->scan($phpcsFile, $tokens, $phpcsFile->findPrevious([T_DECLARE], $position - 1)); + } + } + } +} diff --git a/tests/static/Sniffs/Whitespace/MultipleEmptyLinesSniff.php b/tests/static/Sniffs/Whitespace/MultipleEmptyLinesSniff.php new file mode 100644 index 0000000..e1060b7 --- /dev/null +++ b/tests/static/Sniffs/Whitespace/MultipleEmptyLinesSniff.php @@ -0,0 +1,51 @@ +getTokens(); + if ($phpcsFile->hasCondition($stackPtr, T_FUNCTION) + || $phpcsFile->hasCondition($stackPtr, T_CLASS) + || $phpcsFile->hasCondition($stackPtr, T_INTERFACE) + ) { + if ($tokens[($stackPtr - 1)]['line'] < $tokens[$stackPtr]['line'] + && $tokens[($stackPtr - 2)]['line'] === $tokens[($stackPtr - 1)]['line'] + ) { + // This is an empty line and the line before this one is not + // empty, so this could be the start of a multiple empty line block + $next = $phpcsFile->findNext(T_WHITESPACE, $stackPtr, null, true); + $lines = $tokens[$next]['line'] - $tokens[$stackPtr]['line']; + if ($lines > 1) { + $error = 'Code must not contain multiple empty lines in a row; found %s empty lines'; + $data = [$lines]; + $phpcsFile->addError($error, $stackPtr, 'MultipleEmptyLines', $data); + } + } + } + } +} diff --git a/tests/static/phpcs-ruleset.xml b/tests/static/phpcs-ruleset.xml new file mode 100644 index 0000000..b5ae2aa --- /dev/null +++ b/tests/static/phpcs-ruleset.xml @@ -0,0 +1,30 @@ + + + + Custom Magento ECE Tools coding standard. + + + + + + + + + + + + + + + + + + + _files + _file + diff --git a/tests/static/phpmd-ruleset.xml b/tests/static/phpmd-ruleset.xml new file mode 100644 index 0000000..0eb30e2 --- /dev/null +++ b/tests/static/phpmd-ruleset.xml @@ -0,0 +1,48 @@ + + + + Magento Cloud Code Check Rules + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + _files + diff --git a/tests/unit/.gitignore b/tests/unit/.gitignore new file mode 100644 index 0000000..319b382 --- /dev/null +++ b/tests/unit/.gitignore @@ -0,0 +1 @@ +/phpunit.xml diff --git a/tests/unit/phpunit.xml.dist b/tests/unit/phpunit.xml.dist new file mode 100644 index 0000000..de58085 --- /dev/null +++ b/tests/unit/phpunit.xml.dist @@ -0,0 +1,27 @@ + + + + + + ../../src/Test/Unit + + + + + ../../src + + ../../src/Test + + + + + + + + From cc6ed12a5645f3faa1e3f215997aea361a2cc8f3 Mon Sep 17 00:00:00 2001 From: Oleh Posyniak Date: Mon, 21 Oct 2019 12:42:18 -0500 Subject: [PATCH 2/8] MAGECLOUD-4458: De-compose All Patches from ECE-Tools --- .github/ISSUE_TEMPLATE.md | 35 ++++++++++++++++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 37 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..12ad4e4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,35 @@ + + +### Preconditions + +1. +2. + +### Steps to reproduce + +1. +2. +3. + +### Expected result + +1. [Screenshots, logs or description] + +### Actual result + +1. [Screenshots, logs or description] diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..7e375d4 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,37 @@ + + + + +### Description + + +### Fixed Issues (if relevant) + +1. magento/ece-tools#: Issue title +2. ... + +### Manual testing scenarios + +1. ... +2. ... + +### Contribution checklist + - [ ] Pull request has a meaningful description of its purpose + - [ ] All commits are accompanied by meaningful commit messages + - [ ] All new or changed code is covered with unit/integration tests (if applicable) + - [ ] All automated tests passed successfully (all builds on Travis CI are green) From e480b670d8fd14ff183def73b6a70578f2efb080 Mon Sep 17 00:00:00 2001 From: Oleh Posyniak Date: Mon, 21 Oct 2019 12:43:01 -0500 Subject: [PATCH 3/8] MAGECLOUD-4458: De-compose All Patches from ECE-Tools --- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7e375d4..af5ee0b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -16,10 +16,10 @@ ### Fixed Issues (if relevant) -1. magento/ece-tools#: Issue title +1. magento/magento-cloud-patches#: Issue title 2. ... ### Manual testing scenarios From 745bb229a464216983a24867644dcfb38b754e4d Mon Sep 17 00:00:00 2001 From: Oleh Posyniak Date: Mon, 21 Oct 2019 14:03:40 -0500 Subject: [PATCH 4/8] MAGECLOUD-4458: De-compose All Patches from ECE-Tools --- src/Command/Apply.php | 1 - src/Command/Patch/Manager.php | 21 --------------------- src/Test/Unit/Command/ApplyTest.php | 8 ++++---- src/Test/Unit/Command/Patch/ManagerTest.php | 20 -------------------- 4 files changed, 4 insertions(+), 46 deletions(-) diff --git a/src/Command/Apply.php b/src/Command/Apply.php index 63f9659..e8908bf 100644 --- a/src/Command/Apply.php +++ b/src/Command/Apply.php @@ -65,7 +65,6 @@ protected function configure() */ public function execute(InputInterface $input, OutputInterface $output) { - $this->manager->copyStaticFile($output); $this->manager->applyComposerPatches($input, $output); $this->manager->applyHotFixes($input, $output); } diff --git a/src/Command/Patch/Manager.php b/src/Command/Patch/Manager.php index 9fd0f0e..a61b588 100644 --- a/src/Command/Patch/Manager.php +++ b/src/Command/Patch/Manager.php @@ -65,27 +65,6 @@ public function __construct( $this->directoryList = $directoryList; } - /** - * Copying static file endpoint. - * This resolves issue MAGECLOUD-314. - * - * @param OutputInterface $output - */ - public function copyStaticFile(OutputInterface $output) - { - $magentoRoot = $this->directoryList->getMagentoRoot(); - - if (!$this->filesystem->exists($magentoRoot . '/pub/static.php')) { - $output->writeln('File "static.php" was not found'); - - return; - } - - $this->filesystem->copy($magentoRoot . '/pub/static.php', $magentoRoot . '/pub/front-static.php'); - - $output->writeln('File "static.php" was copied'); - } - /** * Applies patches from composer.json file. * Patches are applying from top to bottom of config list. diff --git a/src/Test/Unit/Command/ApplyTest.php b/src/Test/Unit/Command/ApplyTest.php index 4135571..c751f65 100644 --- a/src/Test/Unit/Command/ApplyTest.php +++ b/src/Test/Unit/Command/ApplyTest.php @@ -7,9 +7,10 @@ namespace Magento\CloudPatches\Test\Unit\Command; -use Illuminate\Contracts\Filesystem\FileNotFoundException; use Magento\CloudPatches\Command\Apply; use Magento\CloudPatches\Command\Patch\Manager; +use Magento\CloudPatches\Command\Patch\ManagerException; +use Magento\CloudPatches\Patch\ApplierException; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; use Symfony\Component\Console\Input\InputInterface; @@ -43,7 +44,8 @@ protected function setUp() } /** - * @throws FileNotFoundException + * @throws ManagerException + * @throws ApplierException */ public function testExecute() { @@ -52,8 +54,6 @@ public function testExecute() /** @var OutputInterface|MockObject $outputMock */ $outputMock = $this->getMockForAbstractClass(OutputInterface::class); - $this->managerMock->expects($this->once()) - ->method('copyStaticFile'); $this->managerMock->expects($this->once()) ->method('applyComposerPatches'); $this->managerMock->expects($this->once()) diff --git a/src/Test/Unit/Command/Patch/ManagerTest.php b/src/Test/Unit/Command/Patch/ManagerTest.php index 0342031..b6baa95 100644 --- a/src/Test/Unit/Command/Patch/ManagerTest.php +++ b/src/Test/Unit/Command/Patch/ManagerTest.php @@ -76,26 +76,6 @@ protected function setUp() ); } - public function testCopyStaticFiles() - { - $this->filesystemMock->expects($this->once()) - ->method('exists') - ->with('/pub/static.php') - ->willReturn(true); - $this->filesystemMock->expects($this->once()) - ->method('copy') - ->with('/pub/static.php', '/pub/front-static.php') - ->willReturn(true); - - /** @var OutputInterface|MockObject $outputMock */ - $outputMock = $this->getMockForAbstractClass(OutputInterface::class); - $outputMock->expects($this->once()) - ->method('writeln') - ->with('File "static.php" was copied'); - - $this->manager->copyStaticFile($outputMock); - } - /** * @throws ApplierException * @throws ManagerException From 08ea071b15a66f8580110f03ecd348276bcb5dc0 Mon Sep 17 00:00:00 2001 From: Oleh Posyniak Date: Mon, 21 Oct 2019 14:37:54 -0500 Subject: [PATCH 5/8] MAGECLOUD-4458: De-compose All Patches from ECE-Tools --- composer.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/composer.json b/composer.json index 81a88f0..2c59b63 100644 --- a/composer.json +++ b/composer.json @@ -9,10 +9,9 @@ "ext-json": "*", "composer/composer": "^1.0", "illuminate/filesystem": "^5.5", - "psr/log": "^1.0", - "symfony/config": "^3.4||^4.3", - "symfony/console": "^2.3||^4.0", - "symfony/dependency-injection": "^3.4||^4.3", + "symfony/config": "^3.3||^4.3", + "symfony/console": "^2.6||^4.0", + "symfony/dependency-injection": "^3.3||^4.3", "symfony/process": "^2.1||^4.1" }, "require-dev": { From 4a13cf936ee4af17b62e264b7a7e6c277c55b7f4 Mon Sep 17 00:00:00 2001 From: Oleh Posyniak Date: Mon, 21 Oct 2019 15:17:40 -0500 Subject: [PATCH 6/8] MAGECLOUD-4458: De-compose All Patches from ECE-Tools --- composer.json | 3 +++ src/Shell/ProcessFactory.php | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 2c59b63..b9fcd4f 100644 --- a/composer.json +++ b/composer.json @@ -14,6 +14,9 @@ "symfony/dependency-injection": "^3.3||^4.3", "symfony/process": "^2.1||^4.1" }, + "conflict": { + "symfony/process": "^4.2" + }, "require-dev": { "phpmd/phpmd": "@stable", "phpunit/phpunit": "^6.2", diff --git a/src/Shell/ProcessFactory.php b/src/Shell/ProcessFactory.php index 006184b..e4fc3da 100644 --- a/src/Shell/ProcessFactory.php +++ b/src/Shell/ProcessFactory.php @@ -36,6 +36,9 @@ public function __construct(DirectoryList $directoryList) */ public function create(array $cmd): Process { - return new Process($cmd, $this->directoryList->getMagentoRoot()); + return new Process( + implode(' ', $cmd), + $this->directoryList->getMagentoRoot() + ); } } From ea038ea13a0f7037e50da66007293df622d247dd Mon Sep 17 00:00:00 2001 From: Oleh Posyniak Date: Wed, 23 Oct 2019 09:52:04 -0500 Subject: [PATCH 7/8] MAGECLOUD-4458: De-compose All Patches from ECE-Tools --- src/Patch/Applier.php | 2 -- src/Test/Unit/Patch/ApplierTest.php | 15 --------------- 2 files changed, 17 deletions(-) diff --git a/src/Patch/Applier.php b/src/Patch/Applier.php index d771bde..9b21e94 100644 --- a/src/Patch/Applier.php +++ b/src/Patch/Applier.php @@ -123,7 +123,6 @@ private function processApply(string $path, string $fullName, bool $deployedFrom { try { $this->processFactory->create(['git', 'apply', $path]) - ->disableOutput() ->mustRun(); } catch (ProcessFailedException $exception) { if ($deployedFromGit) { @@ -136,7 +135,6 @@ private function processApply(string $path, string $fullName, bool $deployedFrom try { $this->processFactory->create(['git', 'apply', '--check', '--reverse', $path]) - ->disableOutput() ->mustRun(); } catch (ProcessFailedException $reverseException) { throw new ApplierException( diff --git a/src/Test/Unit/Patch/ApplierTest.php b/src/Test/Unit/Patch/ApplierTest.php index ac488dd..b62668e 100644 --- a/src/Test/Unit/Patch/ApplierTest.php +++ b/src/Test/Unit/Patch/ApplierTest.php @@ -110,9 +110,6 @@ public function testApply(string $path, string $name, string $packageName, strin ->method('create') ->with(['git', 'apply', $path]) ->willReturn($processMock); - $processMock->expects($this->once()) - ->method('disableOutput') - ->willReturnSelf(); $processMock->expects($this->once()) ->method('mustRun'); @@ -147,9 +144,6 @@ public function testApplyFile(string $path, string $expectedLog) ->method('create') ->with(['git', 'apply', $path]) ->willReturn($processMock); - $processMock->expects($this->once()) - ->method('disableOutput') - ->willReturnSelf(); $processMock->expects($this->once()) ->method('mustRun'); @@ -197,9 +191,6 @@ public function testApplyPathNotExists() ->method('create') ->with(['git', 'apply', 'root/path/to/patch']) ->willReturn($processMock); - $processMock->expects($this->once()) - ->method('disableOutput') - ->willReturnSelf(); $processMock->expects($this->once()) ->method('mustRun'); @@ -275,9 +266,6 @@ public function shellMockReverseCallback(array $command): Process // Command was the reverse check, it's all good. /** @var Process|MockObject $result */ $result = $this->createMock(Process::class); - $result->expects($this->once()) - ->method('disableOutput') - ->willReturnSelf(); $result->expects($this->once()) ->method('mustRun'); @@ -286,9 +274,6 @@ public function shellMockReverseCallback(array $command): Process /** @var Process|MockObject $result */ $result = $this->createMock(Process::class); - $result->expects($this->once()) - ->method('disableOutput') - ->willReturnSelf(); $result->expects($this->once()) ->method('mustRun') ->willThrowException(new ProcessFailedException($result)); From 50aa4de8a47738ac57f187bedf1ee2b29d853d17 Mon Sep 17 00:00:00 2001 From: Oleh Posyniak Date: Thu, 24 Oct 2019 14:41:38 -0500 Subject: [PATCH 8/8] MAGECLOUD-4458: De-compose All Patches from ECE-Tools --- composer.json | 3 +- config/services.xml | 2 +- src/Command/Patch/Manager.php | 11 ++-- src/Filesystem/FileNotFoundException.php | 17 ++++++ src/Filesystem/Filesystem.php | 66 +++++++++++++++++++++ src/Patch/Applier.php | 8 +-- src/Test/Unit/Command/Patch/ManagerTest.php | 4 +- src/Test/Unit/Patch/ApplierTest.php | 2 +- 8 files changed, 97 insertions(+), 16 deletions(-) create mode 100644 src/Filesystem/FileNotFoundException.php create mode 100644 src/Filesystem/Filesystem.php diff --git a/composer.json b/composer.json index b9fcd4f..5975081 100644 --- a/composer.json +++ b/composer.json @@ -7,8 +7,7 @@ "require": { "php": "^7.0", "ext-json": "*", - "composer/composer": "^1.0", - "illuminate/filesystem": "^5.5", + "composer/composer": "@stable", "symfony/config": "^3.3||^4.3", "symfony/console": "^2.6||^4.0", "symfony/dependency-injection": "^3.3||^4.3", diff --git a/config/services.xml b/config/services.xml index 1f727f9..24ffbcc 100644 --- a/config/services.xml +++ b/config/services.xml @@ -8,10 +8,10 @@ - + diff --git a/src/Command/Patch/Manager.php b/src/Command/Patch/Manager.php index a61b588..6661756 100644 --- a/src/Command/Patch/Manager.php +++ b/src/Command/Patch/Manager.php @@ -7,11 +7,11 @@ namespace Magento\CloudPatches\Command\Patch; -use Illuminate\Contracts\Filesystem\FileNotFoundException; -use Illuminate\Filesystem\Filesystem; use Magento\CloudPatches\Command\Apply; use Magento\CloudPatches\Filesystem\DirectoryList; use Magento\CloudPatches\Filesystem\FileList; +use Magento\CloudPatches\Filesystem\FileNotFoundException; +use Magento\CloudPatches\Filesystem\Filesystem; use Magento\CloudPatches\Patch\Applier; use Magento\CloudPatches\Patch\ApplierException; use Symfony\Component\Console\Input\InputInterface; @@ -91,14 +91,13 @@ public function __construct( public function applyComposerPatches(InputInterface $input, OutputInterface $output) { try { - $patches = json_decode( - $this->filesystem->get($this->fileList->getPatches()) ?? '', - true - ); + $content = $this->filesystem->get($this->fileList->getPatches()); } catch (FileNotFoundException $exception) { throw new ManagerException($exception->getMessage(), $exception->getCode(), $exception); } + $patches = json_decode($content, true); + if (!$patches) { $output->writeln('Composer patches not found'); diff --git a/src/Filesystem/FileNotFoundException.php b/src/Filesystem/FileNotFoundException.php new file mode 100644 index 0000000..af9e062 --- /dev/null +++ b/src/Filesystem/FileNotFoundException.php @@ -0,0 +1,17 @@ +isFile($path)) { + return file_get_contents($path); + } + + throw new FileNotFoundException("File does not exist at path {$path}"); + } + + /** + * Determine if the given path is a file. + * + * @param string $file + * @return bool + */ + public function isFile(string $file): bool + { + return is_file($file); + } +} diff --git a/src/Patch/Applier.php b/src/Patch/Applier.php index 9b21e94..d7e1d2c 100644 --- a/src/Patch/Applier.php +++ b/src/Patch/Applier.php @@ -8,8 +8,8 @@ namespace Magento\CloudPatches\Patch; use Composer; -use Illuminate\Filesystem\Filesystem; use Magento\CloudPatches\Filesystem\DirectoryList; +use Magento\CloudPatches\Filesystem\Filesystem; use Magento\CloudPatches\Shell\ProcessFactory; use Symfony\Component\Process\Exception\ProcessFailedException; @@ -42,18 +42,18 @@ class Applier * @param Composer\Composer $composer * @param ProcessFactory $processFactory * @param DirectoryList $directoryList - * @param Filesystem $file + * @param Filesystem $filesystem */ public function __construct( Composer\Composer $composer, ProcessFactory $processFactory, DirectoryList $directoryList, - Filesystem $file + Filesystem $filesystem ) { $this->repository = $composer->getRepositoryManager()->getLocalRepository(); $this->processFactory = $processFactory; $this->directoryList = $directoryList; - $this->filesystem = $file; + $this->filesystem = $filesystem; } /** diff --git a/src/Test/Unit/Command/Patch/ManagerTest.php b/src/Test/Unit/Command/Patch/ManagerTest.php index b6baa95..bb2d775 100644 --- a/src/Test/Unit/Command/Patch/ManagerTest.php +++ b/src/Test/Unit/Command/Patch/ManagerTest.php @@ -8,13 +8,13 @@ namespace Magento\CloudPatches\Test\Unit\Command\Patch; use Composer\Package\RootPackageInterface; -use Illuminate\Contracts\Filesystem\FileNotFoundException; -use Illuminate\Filesystem\Filesystem; use Magento\CloudPatches\Command\Apply; use Magento\CloudPatches\Command\Patch\Manager; use Magento\CloudPatches\Command\Patch\ManagerException; use Magento\CloudPatches\Filesystem\DirectoryList; use Magento\CloudPatches\Filesystem\FileList; +use Magento\CloudPatches\Filesystem\FileNotFoundException; +use Magento\CloudPatches\Filesystem\Filesystem; use Magento\CloudPatches\Patch\Applier; use Magento\CloudPatches\Patch\ApplierException; use PHPUnit\Framework\MockObject\MockObject; diff --git a/src/Test/Unit/Patch/ApplierTest.php b/src/Test/Unit/Patch/ApplierTest.php index b62668e..cd2171b 100644 --- a/src/Test/Unit/Patch/ApplierTest.php +++ b/src/Test/Unit/Patch/ApplierTest.php @@ -11,8 +11,8 @@ use Composer\Package\PackageInterface; use Composer\Repository\RepositoryInterface; use Composer\Repository\RepositoryManager; -use Illuminate\Filesystem\Filesystem; use Magento\CloudPatches\Filesystem\DirectoryList; +use Magento\CloudPatches\Filesystem\Filesystem; use Magento\CloudPatches\Patch\Applier; use Magento\CloudPatches\Patch\ApplierException; use Magento\CloudPatches\Shell\ProcessFactory;