Bug#24609402 MYSQLBINLOG --RAW DOES NOT GET ALL LATEST EVENTS

Problem: When mysqlbinlog is used with --raw option, output file
         is not getting flushed till the process is completed.
         With --stop-never option (mysqlbinlog process never completes),
         output file will never contain any events.

Analysis & Fix: --read-from-remote-server is a mysqlbinlog tool option that
                allows the users to read the binlog from a MySQL server
                rather than reading a local file.
                The read binary log event will be processed and
                written to output files in text format. In this case,
                the output file is getting flushed after every event in
                copy_event_cache_to_file_and_reinit() function.

                By default, mysqlbinlog reads binary log files and writes
                events in text format. But the --raw option tells
                mysqlbinlog to write the events into output file directly
                in their original binary format. In this case, the output
                file is not getting flushed until mysqlbinlog downloads all
                the content. This will be more problematic if --stop-never
                option is used (which is used for "live" binlog backup) as
                the mysqlbinlog process will never stop, the output file
                will never get flushed.

Fix: Flush the output file after every event is written to it
     (just like how we flush when --raw is not specified).

Author: Venkatesh Duggirala

client/mysqlbinlog.cc

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -2490,6 +2490,8 @@ static Exit_status dump_remote_log_entries(PRINT_EVENT_INFO *print_event_info,
           ev->temp_buf=0;
           delete ev;
         }
+        /* Flush result_file after every event */
+        fflush(result_file);
       }
       else
       {

mysql-test/suite/binlog/r/binlog_mysqlbinlog_raw.result

@@ -0,0 +1,20 @@
+RESET MASTER;
+include/stop_dump_threads.inc
+# Step-1: Execute some dummy statements.
+CREATE TABLE t1(i int);
+INSERT INTO t1 values (1);
+# Step-2: Execute MYSQL_BINLOG with --stop-never and --raw option.
+# Step-3: Wait till dump thread transfer is completed.
+# Step-4: copy binary log file that is downloaded till now before
+#         stopping dump thread that is serving mysqlbinlog
+#         --stop-never process.
+# Step-5: kill the dump thread serving the mysqlbinlog --stop-never process
+include/stop_dump_threads.inc
+# Step-6: Cleanup the data, so that it is ready for reapply.
+DROP TABLE t1;
+RESET MASTER;
+# Step-7: Apply the data that we got from --stop-never process.
+# Step-8: Check that the data is there.
+include/assert.inc [Check the table t1 exists and contains one tuple with value 1.]
+# Step-9: Cleanup
+DROP TABLE t1;

mysql-test/suite/binlog/t/binlog_mysqlbinlog_raw.test

@@ -0,0 +1,73 @@
+###############################################################################
+# Bug#24609402 MYSQLBINLOG --RAW DOES NOT GET ALL LATEST EVENTS
+#
+# Problem: When mysqlbinlog is used with --raw option, output file
+#          is not getting flushed till the process is completed.
+#          With --stop-never option (mysqlbinlog process never completes),
+#          output file will never contain any event.
+#
+# Steps to reproduce:
+#          1) Execute some dummy statements(a create and an insert).
+#          2) Execute MYSQL_BINLOG with --stop-never and --raw option
+#             and backup the binlog content.
+#          3) Wait till server dump thread transfers all the data.
+#          4) Copy binary log file that is downloaded till now before
+#             stopping dump thread that is serving mysqlbinlog
+#             --stop-never process.
+#          5) kill the dump thread (serving the mysqlbinlog process).
+#          6) Clean up the data on the server (a drop and a reset).
+#          7) Apply binlog content (binlog from Step-2).
+#          8) Check that table and the data in it exists (data from step-1).
+#          9) Cleanup the server.
+###############################################################################
+
+# Test is not specific to any binlog format. Hence Running only for 'row'.
+--source include/have_binlog_format_row.inc
+
+# binlog file name is needed in the test. To use master-bin.000001,
+# RESET MASTER is needed.
+RESET MASTER;
+# kill the dump threads if there any dump threads (may be from previous test)
+--source include/stop_dump_threads.inc
+
+--echo # Step-1: Execute some dummy statements.
+CREATE TABLE t1(i int);
+INSERT INTO t1 values (1);
+
+--echo # Step-2: Execute MYSQL_BINLOG with --stop-never and --raw option.
+--write_file $MYSQL_TMP_DIR/mysqlbinlog_raw.sh
+#!/bin/bash
+(`$MYSQL_BINLOG --raw --read-from-remote-server --stop-never --user=root --host=127.0.0.1 --port=$MASTER_MYPORT --result-file=$MYSQL_TMP_DIR/ master-bin.000001`) < /dev/null > /dev/null 2>&1 &
+EOF
+--chmod 0755 $MYSQL_TMP_DIR/mysqlbinlog_raw.sh
+--exec $MYSQL_TMP_DIR/mysqlbinlog_raw.sh
+
+--echo # Step-3: Wait till dump thread transfer is completed.
+let $wait_condition= SELECT id from information_schema.processlist where processlist.command like '%Binlog%' and state like '%Master has sent%';
+--source include/wait_condition.inc
+
+--echo # Step-4: copy binary log file that is downloaded till now before
+--echo #         stopping dump thread that is serving mysqlbinlog
+--echo #         --stop-never process.
+--copy_file $MYSQL_TMP_DIR/master-bin.000001 $MYSQL_TMP_DIR/master-bin-back_up_file.000001
+
+--echo # Step-5: kill the dump thread serving the mysqlbinlog --stop-never process
+--source include/stop_dump_threads.inc
+
+--echo # Step-6: Cleanup the data, so that it is ready for reapply.
+DROP TABLE t1;
+RESET MASTER;
+
+--echo # Step-7: Apply the data that we got from --stop-never process.
+--exec $MYSQL_BINLOG $MYSQL_TMP_DIR/master-bin-back_up_file.000001 | $MYSQL --user=root --protocol=tcp --host=127.0.0.1 --port=$MASTER_MYPORT
+
+--echo # Step-8: Check that the data is there.
+--let $assert_text= Check the table t1 exists and contains one tuple with value 1.
+--let $assert_cond= [SELECT COUNT(*) AS Val FROM t1 WHERE i = 1, Val, 1] = 1
+--source include/assert.inc
+
+--echo # Step-9: Cleanup
+DROP TABLE t1;
+--remove_file $MYSQL_TMP_DIR/mysqlbinlog_raw.sh
+--remove_file $MYSQL_TMP_DIR/master-bin.000001
+--remove_file $MYSQL_TMP_DIR/master-bin-back_up_file.000001