diff --git a/contrib/hstore/Makefile b/contrib/hstore/Makefile index 872ca03cd1fb0..72376d9007633 100644 --- a/contrib/hstore/Makefile +++ b/contrib/hstore/Makefile @@ -15,7 +15,7 @@ DATA = hstore--1.4.sql \ hstore--1.5--1.6.sql \ hstore--1.4--1.5.sql \ hstore--1.3--1.4.sql hstore--1.2--1.3.sql \ - hstore--1.1--1.2.sql hstore--1.0--1.1.sql + hstore--1.1--1.2.sql PGFILEDESC = "hstore - key/value pair data type" HEADERS = hstore.h diff --git a/contrib/hstore/hstore--1.0--1.1.sql b/contrib/hstore/hstore--1.0--1.1.sql deleted file mode 100644 index 4e32a575c5f68..0000000000000 --- a/contrib/hstore/hstore--1.0--1.1.sql +++ /dev/null @@ -1,7 +0,0 @@ -/* contrib/hstore/hstore--1.0--1.1.sql */ - --- complain if script is sourced in psql, rather than via ALTER EXTENSION -\echo Use "ALTER EXTENSION hstore UPDATE TO '1.1'" to load this file. \quit - -ALTER EXTENSION hstore DROP OPERATOR => (text, text); -DROP OPERATOR => (text, text); diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 26fda20d19394..fc329c5cff968 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -10226,7 +10226,8 @@ SCRAM-SHA-256$<iteration count>:&l and general database objects (identified by class OID and object OID, in the same way as in pg_description or pg_depend). Also, the right to extend a - relation is represented as a separate lockable object. + relation is represented as a separate lockable object, as is the right to + update pg_database.datfrozenxid. Also, advisory locks can be taken on numbers that have user-defined meanings. @@ -10254,6 +10255,7 @@ SCRAM-SHA-256$<iteration count>:&l Type of the lockable object: relation, extend, + frozenid, page, tuple, transactionid, diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 7dcddf478a112..304c49f07b76b 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1742,6 +1742,12 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser extend Waiting to extend a relation. + + frozenid + Waiting to + update pg_database.datfrozenxid + and pg_database.datminmxid. + object Waiting to acquire a lock on a non-relation database object. @@ -1910,6 +1916,11 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser NotifyQueue Waiting to read or update NOTIFY messages. + + NotifyQueueTail + Waiting to update limit on NOTIFY message + storage. + NotifySLRU Waiting to access the NOTIFY message SLRU @@ -2086,6 +2097,11 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser WALWrite Waiting for WAL buffers to be written to disk. + + WrapLimitsVacuum + Waiting to update limits on transaction id and multixact + consumption. + XactBuffer Waiting for I/O on a transaction status SLRU buffer. diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index e246efbdb5207..aa0b27c9f300f 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -29,51 +29,51 @@ PostgreSQL documentation Description - pg_basebackup is used to take base backups of - a running PostgreSQL database cluster. These - are taken without affecting other clients to the database, and can be used + pg_basebackup is used to take a base backup of + a running PostgreSQL database cluster. The backup + is taken without affecting other clients of the database, and can be used both for point-in-time recovery (see ) - and as the starting point for a log shipping or streaming replication standby - servers (see ). + and as the starting point for a log-shipping or streaming-replication standby + server (see ). - pg_basebackup makes a binary copy of the database - cluster files, while making sure the system is put in and + pg_basebackup makes an exact copy of the database + cluster's files, while making sure the server is put into and out of backup mode automatically. Backups are always taken of the entire database cluster; it is not possible to back up individual databases or - database objects. For individual database backups, a tool such as + database objects. For selective backups, another tool such as must be used. The backup is made over a regular PostgreSQL - connection, and uses the replication protocol. The connection must be made - with a user having REPLICATION permissions - (see ) or a superuser, - and pg_hba.conf must explicitly permit the replication - connection. The server must also be configured - with set high enough to leave at least - one session available for the backup and one for WAL streaming (if used). + connection that uses the replication protocol. The connection must be made + with a user ID that has REPLICATION permissions + (see ) or is a superuser, + and pg_hba.conf + must permit the replication connection. The server must also be configured + with set high enough to provide at + least one walsender for the backup plus one for WAL streaming (if used). - There can be multiple pg_basebackups running at the same time, but it is + There can be multiple pg_basebackups running at the same time, but it is usually better from a performance point of view to take only one backup, and copy the result. pg_basebackup can make a base backup from - not only the primary but also the standby. To take a backup from the standby, + not only a primary server but also a standby. To take a backup from a standby, set up the standby so that it can accept replication connections (that is, set max_wal_senders and , - and configure host-based authentication). + and configure its pg_hba.conf appropriately). You will also need to enable on the primary. - Note that there are some limitations in an online backup from the standby: + Note that there are some limitations in taking a backup from a standby: @@ -89,7 +89,7 @@ PostgreSQL documentation - If the standby is promoted to the primary during online backup, the backup fails. + If the standby is promoted to be primary during backup, the backup fails. @@ -105,7 +105,7 @@ PostgreSQL documentation Whenever pg_basebackup is taking a base - backup, the pg_stat_progress_basebackup + backup, the server's pg_stat_progress_basebackup view will report the progress of the backup. See for details. @@ -116,7 +116,7 @@ PostgreSQL documentation The following command-line options control the location and format of the - output. + output: @@ -124,15 +124,15 @@ PostgreSQL documentation - Directory to write the output to. - pg_basebackup will create the directory and - any parent directories if necessary. The directory may already exist, - but it is an error if the directory already exists and is not empty. + Sets the target directory to write the output to. + pg_basebackup will create this directory + (and any missing parent directories) if it does not exist. If it + already exists, it must be empty. - When the backup is in tar mode, and the directory is specified as - - (dash), the tar file will be written to - stdout. + When the backup is in tar format, the target directory may be + specified as - (dash), causing the tar file to be + written to stdout. This option is required. @@ -155,12 +155,12 @@ PostgreSQL documentation Write the output as plain files, with the same layout as the - current data directory and tablespaces. When the cluster has + source server's data directory and tablespaces. When the cluster has no additional tablespaces, the whole database will be placed in the target directory. If the cluster contains additional tablespaces, the main data directory will be placed in the target directory, but all other tablespaces will be placed - in the same absolute path as they have on the server. + in the same absolute path as they have on the source server. This is the default format. @@ -174,15 +174,15 @@ PostgreSQL documentation Write the output as tar files in the target directory. The main - data directory will be written to a file named - base.tar, and all other tablespaces will - be named after the tablespace OID. - + data directory's contents will be written to a file named + base.tar, and each other tablespace will be + written to a separate tar file named after that tablespace's OID. + - If the value - (dash) is specified as - target directory, the tar contents will be written to - standard output, suitable for piping to for example - gzip. This is only possible if + If the target directory is specified as - + (dash), the tar contents will be written to + standard output, suitable for piping to (for example) + gzip. This is only allowed if the cluster has no additional tablespaces and WAL streaming is not used. @@ -192,40 +192,22 @@ PostgreSQL documentation - - - - - - The maximum transfer rate of data transferred from the server. Values are - in kilobytes per second. Use a suffix of M to indicate megabytes - per second. A suffix of k is also accepted, and has no effect. - Valid values are between 32 kilobytes per second and 1024 megabytes per second. - - - The purpose is to limit the impact of pg_basebackup - on the running server. - - - This option always affects transfer of the data directory. Transfer of - WAL files is only affected if the collection method is fetch. - - - - - Create standby.signal and append connection settings - to postgresql.auto.conf in the output - directory (or into the base archive file when using tar format) to - ease setting up a standby server. + Creates a standby.signal file and appends + connection settings to the postgresql.auto.conf + file in the target directory (or within the base archive file when + using tar format). This eases setting up a standby server using the + results of the backup. + + The postgresql.auto.conf file will record the connection settings and, if specified, the replication slot - that pg_basebackup is using, so that the + that pg_basebackup is using, so that streaming replication will use the same settings later on. @@ -237,17 +219,21 @@ PostgreSQL documentation - Relocate the tablespace in directory olddir + Relocates the tablespace in directory olddir to newdir during the backup. To be effective, olddir must exactly match the - path specification of the tablespace as it is currently defined. (But - it is not an error if there is no tablespace - in olddir contained in the backup.) + path specification of the tablespace as it is defined on the source + server. (But it is not an error if there is no tablespace + in olddir on the source server.) + Meanwhile newdir is a directory in the + receiving host's filesystem. As with the main target directory, + newdir need not exist already, but if + it does exist it must be empty. Both olddir - and newdir must be absolute paths. If a - path happens to contain a = sign, escape it with a - backslash. This option can be specified multiple times for multiple - tablespaces. See examples below. + and newdir must be absolute paths. If + either path needs to contain an equal sign (=), + precede that with a backslash. This option can be specified multiple + times for multiple tablespaces. @@ -263,10 +249,16 @@ PostgreSQL documentation - Specifies the location for the write-ahead log directory. + Sets the directory to write WAL (write-ahead log) files to. + By default WAL files will be placed in + the pg_wal subdirectory of the target + directory, but this option can be used to place them elsewhere. waldir must be an absolute path. - The write-ahead log directory can only be specified when - the backup is in plain mode. + As with the main target directory, + waldir need not exist already, but if + it does exist it must be empty. + This option can only be specified when + the backup is in plain format. @@ -276,16 +268,16 @@ PostgreSQL documentation - Includes the required write-ahead log files (WAL files) in the + Includes the required WAL (write-ahead log) files in the backup. This will include all write-ahead logs generated during the backup. Unless the method none is specified, - it is possible to start a postmaster directly in the extracted + it is possible to start a postmaster in the target directory without the need to consult the log archive, thus - making this a completely standalone backup. + making the output a completely standalone backup. - The following methods for collecting the write-ahead logs are - supported: + The following methods for collecting the + write-ahead logs are supported: @@ -293,7 +285,7 @@ PostgreSQL documentation none - Don't include write-ahead log in the backup. + Don't include write-ahead logs in the backup. @@ -304,15 +296,16 @@ PostgreSQL documentation The write-ahead log files are collected at the end of the backup. - Therefore, it is necessary for the + Therefore, it is necessary for the source server's parameter to be set high - enough that the log is not removed before the end of the backup. - If the log has been rotated when it's time to transfer it, the - backup will fail and be unusable. + enough that the required log data is not removed before the end + of the backup. If the required log data has been recycled + before it's time to transfer it, the backup will fail and be + unusable. - When tar format mode is used, the write-ahead log files will be - written to the base.tar file. + When tar format is used, the write-ahead log files will be + included in the base.tar file. @@ -322,16 +315,16 @@ PostgreSQL documentation stream - Stream the write-ahead log while the backup is created. This will - open a second connection to the server and start streaming the - write-ahead log in parallel while running the backup. Therefore, - it will use up two connections configured by the - parameter. As long as the - client can keep up with write-ahead log received, using this mode - requires no extra write-ahead logs to be saved on the primary. + Stream write-ahead log data while the backup is being taken. + This method will open a second connection to the server and + start streaming the write-ahead log in parallel while running + the backup. Therefore, it will require two replication + connections not just one. As long as the client can keep up + with the write-ahead log data, using this method requires no + extra write-ahead logs to be saved on the source server. - When tar format mode is used, the write-ahead log files will be + When tar format is used, the write-ahead log files will be written to a separate file named pg_wal.tar (if the server is a version earlier than 10, the file will be named pg_xlog.tar). @@ -375,7 +368,7 @@ PostgreSQL documentation The following command-line options control the generation of the - backup and the running of the program. + backup and the running of the program: @@ -383,7 +376,8 @@ PostgreSQL documentation - Sets checkpoint mode to fast (immediate) or spread (default) (see ). + Sets checkpoint mode to fast (immediate) or spread (the default) + (see ). @@ -393,9 +387,9 @@ PostgreSQL documentation - This option causes creation of a replication slot named by the - --slot option before starting the backup. - An error is raised if the slot already exists. + Specifies that the replication slot named by the + --slot option should be created before starting + the backup. An error is raised if the slot already exists. @@ -418,9 +412,9 @@ PostgreSQL documentation By default, when pg_basebackup aborts with an error, it removes any directories it might have created before - discovering that it cannot finish the job (for example, data directory - and write-ahead log directory). This option inhibits tidying-up and is - thus useful for debugging. + discovering that it cannot finish the job (for example, the target + directory and write-ahead log directory). This option inhibits + tidying-up and is thus useful for debugging. @@ -460,19 +454,41 @@ PostgreSQL documentation + + + + + + Sets the maximum transfer rate at which data is collected from the + source server. This can be useful to limit the impact + of pg_basebackup on the server. Values + are in kilobytes per second. Use a suffix of M + to indicate megabytes per second. A suffix of k + is also accepted, and has no effect. Valid values are between 32 + kilobytes per second and 1024 megabytes per second. + + + This option always affects transfer of the data directory. Transfer of + WAL files is only affected if the collection method + is fetch. + + + + This option can only be used together with -X - stream. It causes the WAL streaming to use the specified + stream. It causes WAL streaming to use the specified replication slot. If the base backup is intended to be used as a - streaming replication standby using replication slots, it should then - use the same replication slot name - in . That way, it is ensured that - the server does not remove any necessary WAL data in the time between - the end of the base backup and the start of streaming replication. + streaming-replication standby using a replication slot, the standby + should then use the same replication slot name as + . This ensures that the + primary server does not remove any necessary WAL data in the time + between the end of the base backup and the start of streaming + replication on the new standby. The specified replication slot has to exist unless the @@ -522,15 +538,15 @@ PostgreSQL documentation Using a SHA hash function provides a cryptographically secure digest of each file for users who wish to verify that the backup has not been - tampered with, while the CRC32C algorithm provides a checksum which is - much faster to calculate and good at catching errors due to accidental + tampered with, while the CRC32C algorithm provides a checksum that is + much faster to calculate; it is good at catching errors due to accidental changes but is not resistant to targeted modifications. Note that, to be useful against an adversary who has access to the backup, the backup manifest would need to be stored securely elsewhere or otherwise verified not to have been modified since the backup was taken. - can be used to check the + can be used to check the integrity of a backup against the backup manifest. @@ -552,11 +568,11 @@ PostgreSQL documentation - This option prevents the server from estimating the total + Prevents the server from estimating the total amount of backup data that will be streamed, resulting in the - backup_total column in the - pg_stat_progress_basebackup - to be NULL. + backup_total column in the + pg_stat_progress_basebackup view + always being NULL. Without this option, the backup will start by enumerating @@ -578,7 +594,7 @@ PostgreSQL documentation Disables generation of a backup manifest. If this option is not specified, the server will generate and send a backup manifest - which can be verified using . + which can be verified using . The manifest is a list of every file present in the backup with the exception of any WAL files that may be included. It also stores the size, last modification time, and an optional checksum for each file. @@ -590,16 +606,17 @@ PostgreSQL documentation - This option prevents the creation of a temporary replication slot - during the backup even if it's supported by the server. + Prevents the creation of a temporary replication slot + for the backup. - Temporary replication slots are created by default if no slot name - is given with the option when using log streaming. + By default, if log streaming is selected but no slot name is given + with the option, then a temporary replication + slot is created (if supported by the source server). The main purpose of this option is to allow taking a base backup when - the server is out of free replication slots. Using replication slots + the server has no free replication slots. Using a replication slot is almost always preferred, because it prevents needed WAL from being removed by the server during the backup. @@ -617,7 +634,7 @@ PostgreSQL documentation By default, checksums are verified and checksum failures will result in a non-zero exit status. However, the base backup will not be removed in such a case, as if the option - had been used. Checksum verifications failures will also be reported + had been used. Checksum verification failures will also be reported in the pg_stat_database view. @@ -627,7 +644,8 @@ PostgreSQL documentation - The following command-line options control the database connection parameters. + The following command-line options control the connection to the source + server: @@ -641,7 +659,7 @@ PostgreSQL documentation The option is called --dbname for consistency with other client applications, but because pg_basebackup - doesn't connect to any particular database in the cluster, database + doesn't connect to any particular database in the cluster, any database name in the connection string will be ignored. @@ -654,7 +672,7 @@ PostgreSQL documentation Specifies the host name of the machine on which the server is running. If the value begins with a slash, it is used as the - directory for the Unix domain socket. The default is taken + directory for a Unix domain socket. The default is taken from the PGHOST environment variable, if set, else a Unix domain socket connection is attempted. @@ -679,11 +697,12 @@ PostgreSQL documentation - Specifies the number of seconds between status packets sent back to the - server. This allows for easier monitoring of the progress from server. - A value of zero disables the periodic status updates completely, + Specifies the number of seconds between status packets sent back to + the source server. Smaller values allow more accurate monitoring of + backup progress from the server. + A value of zero disables periodic status updates completely, although an update will still be sent when requested by the server, to - avoid timeout disconnect. The default value is 10 seconds. + avoid timeout-based disconnects. The default value is 10 seconds. @@ -693,7 +712,7 @@ PostgreSQL documentation - User name to connect as. + Specifies the user name to connect as. @@ -703,7 +722,7 @@ PostgreSQL documentation - Never issue a password prompt. If the server requires + Prevents issuing a password prompt. If the server requires password authentication and a password is not available by other means such as a .pgpass file, the connection attempt will fail. This option can be useful in @@ -718,8 +737,8 @@ PostgreSQL documentation - Force pg_basebackup to prompt for a - password before connecting to a database. + Forces pg_basebackup to prompt for a + password before connecting to the source server. @@ -745,7 +764,7 @@ PostgreSQL documentation - Print the pg_basebackup version and exit. + Prints the pg_basebackup version and exits. @@ -755,8 +774,8 @@ PostgreSQL documentation - Show help about pg_basebackup command line - arguments, and exit. + Shows help about pg_basebackup command line + arguments, and exits. @@ -787,11 +806,10 @@ PostgreSQL documentation Notes - At the beginning of the backup, a checkpoint needs to be written on the - server the backup is taken from. Especially if the option - --checkpoint=fast is not used, this can take some time - during which pg_basebackup will be appear - to be idle. + At the beginning of the backup, a checkpoint needs to be performed on the + source server. This can take some time (especially if the option + --checkpoint=fast is not used), during + which pg_basebackup will appear to be idle. @@ -806,8 +824,8 @@ PostgreSQL documentation - Tablespaces will in plain format by default be backed up to the same path - they have on the server, unless the + In plain format, tablespaces will be backed up to the same path + they have on the source server, unless the option --tablespace-mapping is used. Without this option, running a plain format base backup on the same host as the server will not work if tablespaces are in use, because the backup would @@ -816,8 +834,9 @@ PostgreSQL documentation - When tar format mode is used, it is the user's responsibility to unpack each - tar file before starting the PostgreSQL server. If there are additional tablespaces, the + When tar format is used, it is the user's responsibility to unpack each + tar file before starting a PostgreSQL server that uses the data. If there + are additional tablespaces, the tar files for them need to be unpacked in the correct locations. In this case the symbolic links for those tablespaces will be created by the server according to the contents of the tablespace_map file that is @@ -827,15 +846,14 @@ PostgreSQL documentation pg_basebackup works with servers of the same or an older major version, down to 9.1. However, WAL streaming mode (-X - stream) only works with server version 9.3 and later, and tar format mode - (--format=tar) of the current version only works with server version 9.5 - or later. + stream) only works with server version 9.3 and later, and tar format + (--format=tar) only works with server version 9.5 + and later. - pg_basebackup will preserve group permissions in - both the plain and tar formats if group - permissions are enabled on the source cluster. + pg_basebackup will preserve group permissions + for data files if group permissions are enabled on the source cluster. diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index f75e1cf0e7b0f..9b5f417eac442 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -6920,8 +6920,6 @@ HeapTupleHeaderAdvanceLatestRemovedXid(HeapTupleHeader tuple, * updated/deleted by the inserting transaction. * * Look for a committed hint bit, or if no xmin bit is set, check clog. - * This needs to work on both primary and standby, where it is used to - * assess btree delete records. */ if (HeapTupleHeaderXminCommitted(tuple) || (!HeapTupleHeaderXminInvalid(tuple) && TransactionIdDidCommit(xmin))) diff --git a/src/backend/access/transam/README b/src/backend/access/transam/README index c5f09667ba159..1edc8180c1284 100644 --- a/src/backend/access/transam/README +++ b/src/backend/access/transam/README @@ -635,12 +635,11 @@ be reconstructed later following a crash and the action is simply a way of optimising for performance. When a hint is written we use MarkBufferDirtyHint() to mark the block dirty. -If the buffer is clean and checksums are in use then -MarkBufferDirtyHint() inserts an XLOG_FPI record to ensure that we -take a full page image that includes the hint. We do this to avoid -a partial page write, when we write the dirtied page. WAL is not -written during recovery, so we simply skip dirtying blocks because -of hints when in recovery. +If the buffer is clean and checksums are in use then MarkBufferDirtyHint() +inserts an XLOG_FPI_FOR_HINT record to ensure that we take a full page image +that includes the hint. We do this to avoid a partial page write, when we +write the dirtied page. WAL is not written during recovery, so we simply skip +dirtying blocks because of hints when in recovery. If you do decide to optimise away a WAL record, then any calls to MarkBufferDirty() must be replaced by MarkBufferDirtyHint(), diff --git a/src/backend/access/transam/slru.c b/src/backend/access/transam/slru.c index d1dbb43e096c1..7640f153c227b 100644 --- a/src/backend/access/transam/slru.c +++ b/src/backend/access/transam/slru.c @@ -1191,6 +1191,14 @@ SimpleLruFlush(SlruCtl ctl, bool allow_redirtied) /* * Remove all segments before the one holding the passed page number + * + * All SLRUs prevent concurrent calls to this function, either with an LWLock + * or by calling it only as part of a checkpoint. Mutual exclusion must begin + * before computing cutoffPage. Mutual exclusion must end after any limit + * update that would permit other backends to write fresh data into the + * segment immediately preceding the one containing cutoffPage. Otherwise, + * when the SLRU is quite full, SimpleLruTruncate() might delete that segment + * after it has accrued freshly-written data. */ void SimpleLruTruncate(SlruCtl ctl, int cutoffPage) diff --git a/src/backend/access/transam/subtrans.c b/src/backend/access/transam/subtrans.c index a087a5554210c..a50f60b99af28 100644 --- a/src/backend/access/transam/subtrans.c +++ b/src/backend/access/transam/subtrans.c @@ -349,8 +349,8 @@ ExtendSUBTRANS(TransactionId newestXact) /* * Remove all SUBTRANS segments before the one holding the passed transaction ID * - * This is normally called during checkpoint, with oldestXact being the - * oldest TransactionXmin of any running transaction. + * oldestXact is the oldest TransactionXmin of any running transaction. This + * is called only during checkpoint. */ void TruncateSUBTRANS(TransactionId oldestXact) diff --git a/src/backend/access/transam/varsup.c b/src/backend/access/transam/varsup.c index 2d2b05be36c47..a4944faa32e34 100644 --- a/src/backend/access/transam/varsup.c +++ b/src/backend/access/transam/varsup.c @@ -367,12 +367,13 @@ SetTransactionIdLimit(TransactionId oldest_datfrozenxid, Oid oldest_datoid) * We'll refuse to continue assigning XIDs in interactive mode once we get * within 3M transactions of data loss. This leaves lots of room for the * DBA to fool around fixing things in a standalone backend, while not - * being significant compared to total XID space. (Note that since - * vacuuming requires one transaction per table cleaned, we had better be - * sure there's lots of XIDs left...) Also, at default BLCKSZ, this - * leaves two completely-idle segments. In the event of edge-case bugs - * involving page or segment arithmetic, idle segments render the bugs - * unreachable outside of single-user mode. + * being significant compared to total XID space. (VACUUM requires an XID + * if it truncates at wal_level!=minimal. "VACUUM (ANALYZE)", which a DBA + * might do by reflex, assigns an XID. Hence, we had better be sure + * there's lots of XIDs left...) Also, at default BLCKSZ, this leaves two + * completely-idle segments. In the event of edge-case bugs involving + * page or segment arithmetic, idle segments render the bugs unreachable + * outside of single-user mode. */ xidStopLimit = xidWrapLimit - 3000000; if (xidStopLimit < FirstNormalTransactionId) diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c index 71b7577afc067..774b26fd2c4d2 100644 --- a/src/backend/commands/async.c +++ b/src/backend/commands/async.c @@ -244,19 +244,22 @@ typedef struct QueueBackendStatus /* * Shared memory state for LISTEN/NOTIFY (excluding its SLRU stuff) * - * The AsyncQueueControl structure is protected by the NotifyQueueLock. + * The AsyncQueueControl structure is protected by the NotifyQueueLock and + * NotifyQueueTailLock. * - * When holding the lock in SHARED mode, backends may only inspect their own - * entries as well as the head and tail pointers. Consequently we can allow a - * backend to update its own record while holding only SHARED lock (since no - * other backend will inspect it). + * When holding NotifyQueueLock in SHARED mode, backends may only inspect + * their own entries as well as the head and tail pointers. Consequently we + * can allow a backend to update its own record while holding only SHARED lock + * (since no other backend will inspect it). * - * When holding the lock in EXCLUSIVE mode, backends can inspect the entries - * of other backends and also change the head and tail pointers. + * When holding NotifyQueueLock in EXCLUSIVE mode, backends can inspect the + * entries of other backends and also change the head pointer. When holding + * both NotifyQueueLock and NotifyQueueTailLock in EXCLUSIVE mode, backends + * can change the tail pointer. * * NotifySLRULock is used as the control lock for the pg_notify SLRU buffers. - * In order to avoid deadlocks, whenever we need both locks, we always first - * get NotifyQueueLock and then NotifySLRULock. + * In order to avoid deadlocks, whenever we need multiple locks, we first get + * NotifyQueueTailLock, then NotifyQueueLock, and lastly NotifySLRULock. * * Each backend uses the backend[] array entry with index equal to its * BackendId (which can range from 1 to MaxBackends). We rely on this to make @@ -299,13 +302,10 @@ static SlruCtlData NotifyCtlData; #define QUEUE_FULL_WARN_INTERVAL 5000 /* warn at most once every 5s */ /* - * slru.c currently assumes that all filenames are four characters of hex - * digits. That means that we can use segments 0000 through FFFF. - * Each segment contains SLRU_PAGES_PER_SEGMENT pages which gives us - * the pages from 0 to SLRU_PAGES_PER_SEGMENT * 0x10000 - 1. - * - * It's of course possible to enhance slru.c, but this gives us so much - * space already that it doesn't seem worth the trouble. + * Use segments 0000 through FFFF. Each contains SLRU_PAGES_PER_SEGMENT pages + * which gives us the pages from 0 to SLRU_PAGES_PER_SEGMENT * 0x10000 - 1. + * We could use as many segments as SlruScanDirectory() allows, but this gives + * us so much space already that it doesn't seem worth the trouble. * * The most data we can have in the queue at a time is QUEUE_MAX_PAGE/2 * pages, because more than that would confuse slru.c into thinking there @@ -2177,6 +2177,10 @@ asyncQueueAdvanceTail(void) int newtailpage; int boundary; + /* Restrict task to one backend per cluster; see SimpleLruTruncate(). */ + LWLockAcquire(NotifyQueueTailLock, LW_EXCLUSIVE); + + /* Compute the new tail. */ LWLockAcquire(NotifyQueueLock, LW_EXCLUSIVE); min = QUEUE_HEAD; for (BackendId i = QUEUE_FIRST_LISTENER; i > 0; i = QUEUE_NEXT_LISTENER(i)) @@ -2185,7 +2189,6 @@ asyncQueueAdvanceTail(void) min = QUEUE_POS_MIN(min, QUEUE_BACKEND_POS(i)); } oldtailpage = QUEUE_POS_PAGE(QUEUE_TAIL); - QUEUE_TAIL = min; LWLockRelease(NotifyQueueLock); /* @@ -2205,6 +2208,17 @@ asyncQueueAdvanceTail(void) */ SimpleLruTruncate(NotifyCtl, newtailpage); } + + /* + * Advertise the new tail. This changes asyncQueueIsFull()'s verdict for + * the segment immediately prior to the new tail, allowing fresh data into + * that segment. + */ + LWLockAcquire(NotifyQueueLock, LW_EXCLUSIVE); + QUEUE_TAIL = min; + LWLockRelease(NotifyQueueLock); + + LWLockRelease(NotifyQueueTailLock); } /* diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index aba13c31d1bc2..23eb605d4cb25 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -949,11 +949,11 @@ vacuum_set_xid_limits(Relation rel, /* * We can always ignore processes running lazy vacuum. This is because we * use these values only for deciding which tuples we must keep in the - * tables. Since lazy vacuum doesn't write its XID anywhere, it's safe to - * ignore it. In theory it could be problematic to ignore lazy vacuums in - * a full vacuum, but keep in mind that only one vacuum process can be - * working on a particular table at any time, and that each vacuum is - * always an independent transaction. + * tables. Since lazy vacuum doesn't write its XID anywhere (usually no + * XID assigned), it's safe to ignore it. In theory it could be + * problematic to ignore lazy vacuums in a full vacuum, but keep in mind + * that only one vacuum process can be working on a particular table at + * any time, and that each vacuum is always an independent transaction. */ *oldestXmin = GetOldestNonRemovableTransactionId(rel); @@ -1361,6 +1361,14 @@ vac_update_datfrozenxid(void) bool bogus = false; bool dirty = false; + /* + * Restrict this task to one backend per database. This avoids race + * conditions that would move datfrozenxid or datminmxid backward. It + * avoids calling vac_truncate_clog() with a datfrozenxid preceding a + * datfrozenxid passed to an earlier vac_truncate_clog() call. + */ + LockDatabaseFrozenIds(ExclusiveLock); + /* * Initialize the "min" calculation with * GetOldestNonRemovableTransactionId(), which is a reasonable @@ -1551,6 +1559,9 @@ vac_truncate_clog(TransactionId frozenXID, bool bogus = false; bool frozenAlreadyWrapped = false; + /* Restrict task to one backend per cluster; see SimpleLruTruncate(). */ + LWLockAcquire(WrapLimitsVacuumLock, LW_EXCLUSIVE); + /* init oldest datoids to sync with my frozenXID/minMulti values */ oldestxid_datoid = MyDatabaseId; minmulti_datoid = MyDatabaseId; @@ -1660,6 +1671,8 @@ vac_truncate_clog(TransactionId frozenXID, */ SetTransactionIdLimit(frozenXID, oldestxid_datoid); SetMultiXactIdLimit(minMulti, minmulti_datoid, false); + + LWLockRelease(WrapLimitsVacuumLock); } diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index f1ae6f9f84430..a2a963bd5b41f 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -3578,7 +3578,7 @@ IncrBufferRefCount(Buffer buffer) * This is essentially the same as MarkBufferDirty, except: * * 1. The caller does not write WAL; so if checksums are enabled, we may need - * to write an XLOG_FPI WAL record to protect against torn pages. + * to write an XLOG_FPI_FOR_HINT WAL record to protect against torn pages. * 2. The caller might have only share-lock instead of exclusive-lock on the * buffer's content lock. * 3. This function does not guarantee that the buffer is always marked dirty diff --git a/src/backend/storage/lmgr/lmgr.c b/src/backend/storage/lmgr/lmgr.c index 20103200952e7..7409de9405925 100644 --- a/src/backend/storage/lmgr/lmgr.c +++ b/src/backend/storage/lmgr/lmgr.c @@ -460,6 +460,21 @@ UnlockRelationForExtension(Relation relation, LOCKMODE lockmode) LockRelease(&tag, lockmode, false); } +/* + * LockDatabaseFrozenIds + * + * This allows one backend per database to execute vac_update_datfrozenxid(). + */ +void +LockDatabaseFrozenIds(LOCKMODE lockmode) +{ + LOCKTAG tag; + + SET_LOCKTAG_DATABASE_FROZEN_IDS(tag, MyDatabaseId); + + (void) LockAcquire(&tag, lockmode, false, false); +} + /* * LockPage * @@ -1098,6 +1113,11 @@ DescribeLockTag(StringInfo buf, const LOCKTAG *tag) tag->locktag_field2, tag->locktag_field1); break; + case LOCKTAG_DATABASE_FROZEN_IDS: + appendStringInfo(buf, + _("pg_database.datfrozenxid of database %u"), + tag->locktag_field1); + break; case LOCKTAG_PAGE: appendStringInfo(buf, _("page %u of relation %u of database %u"), diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index e6985e8eedfb1..774292fd94277 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -50,3 +50,6 @@ MultiXactTruncationLock 41 OldSnapshotTimeMapLock 42 LogicalRepWorkerLock 43 XactTruncationLock 44 +# 45 was XactTruncationLock until removal of BackendRandomLock +WrapLimitsVacuumLock 46 +NotifyQueueTailLock 47 diff --git a/src/backend/utils/adt/float.c b/src/backend/utils/adt/float.c index ffd1ce8c76104..429c9280c0cf7 100644 --- a/src/backend/utils/adt/float.c +++ b/src/backend/utils/adt/float.c @@ -271,18 +271,6 @@ float4in(PG_FUNCTION_ARGS) errmsg("invalid input syntax for type %s: \"%s\"", "real", orig_num))); } -#ifdef HAVE_BUGGY_SOLARIS_STRTOD - else - { - /* - * Many versions of Solaris have a bug wherein strtod sets endptr to - * point one byte beyond the end of the string when given "inf" or - * "infinity". - */ - if (endptr != num && endptr[-1] == '\0') - endptr--; - } -#endif /* HAVE_BUGGY_SOLARIS_STRTOD */ /* skip trailing whitespace */ while (*endptr != '\0' && isspace((unsigned char) *endptr)) @@ -499,18 +487,6 @@ float8in_internal_opt_error(char *num, char **endptr_p, type_name, orig_string))), have_error); } -#ifdef HAVE_BUGGY_SOLARIS_STRTOD - else - { - /* - * Many versions of Solaris have a bug wherein strtod sets endptr to - * point one byte beyond the end of the string when given "inf" or - * "infinity". - */ - if (endptr != num && endptr[-1] == '\0') - endptr--; - } -#endif /* HAVE_BUGGY_SOLARIS_STRTOD */ /* skip trailing whitespace */ while (*endptr != '\0' && isspace((unsigned char) *endptr)) diff --git a/src/backend/utils/adt/lockfuncs.c b/src/backend/utils/adt/lockfuncs.c index e992d1bbfcedf..f592292d067b8 100644 --- a/src/backend/utils/adt/lockfuncs.c +++ b/src/backend/utils/adt/lockfuncs.c @@ -29,6 +29,7 @@ const char *const LockTagTypeNames[] = { "relation", "extend", + "frozenid", "page", "tuple", "transactionid", @@ -254,6 +255,17 @@ pg_lock_status(PG_FUNCTION_ARGS) nulls[8] = true; nulls[9] = true; break; + case LOCKTAG_DATABASE_FROZEN_IDS: + values[1] = ObjectIdGetDatum(instance->locktag.locktag_field1); + nulls[2] = true; + nulls[3] = true; + nulls[4] = true; + nulls[5] = true; + nulls[6] = true; + nulls[7] = true; + nulls[8] = true; + nulls[9] = true; + break; case LOCKTAG_PAGE: values[1] = ObjectIdGetDatum(instance->locktag.locktag_field1); values[2] = ObjectIdGetDatum(instance->locktag.locktag_field2); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index d1a0678935397..31e99c2a6da5d 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -611,14 +611,9 @@ XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats) double rec_len_pct, fpi_len_pct; - /* --- - * Make a first pass to calculate column totals: - * count(*), - * sum(xl_len+SizeOfXLogRecord), - * sum(xl_tot_len-xl_len-SizeOfXLogRecord), and - * sum(xl_tot_len). - * These are used to calculate percentages for each record type. - * --- + /* + * Each row shows its percentages of the total, so make a first pass to + * calculate column totals. */ for (ri = 0; ri < RM_NEXT_ID; ri++) diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 9b2da56379e15..4146753d4765d 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -43,11 +43,8 @@ typedef struct XLogPageHeaderData /* * When there is not enough space on current page for whole record, we * continue on the next page. xlp_rem_len is the number of bytes - * remaining from a previous page. - * - * Note that xlp_rem_len includes backup-block data; that is, it tracks - * xl_tot_len not xl_len in the initial header. Also note that the - * continuation data isn't necessarily aligned. + * remaining from a previous page; it tracks xl_tot_len in the initial + * header. Note that the continuation data isn't necessarily aligned. */ uint32 xlp_rem_len; /* total len of remaining data for record */ } XLogPageHeaderData; diff --git a/src/include/port/cygwin.h b/src/include/port/cygwin.h index f1fc1a93d76c0..64d69936e5e02 100644 --- a/src/include/port/cygwin.h +++ b/src/include/port/cygwin.h @@ -1,14 +1,5 @@ /* src/include/port/cygwin.h */ -#include - -/* - * Check for b20.1 and disable AF_UNIX family socket support. - */ -#if CYGWIN_VERSION_DLL_MAJOR < 1001 -#undef HAVE_UNIX_SOCKETS -#endif - #ifdef BUILDING_DLL #define PGDLLIMPORT __declspec (dllexport) #else diff --git a/src/include/port/solaris.h b/src/include/port/solaris.h index eeb1a320bd5b7..e63a3bd824d6d 100644 --- a/src/include/port/solaris.h +++ b/src/include/port/solaris.h @@ -24,15 +24,3 @@ #if defined(__i386__) #include #endif - -/* - * Many versions of Solaris have broken strtod() --- see bug #4751182. - * This has been fixed in current versions of Solaris: - * - * http://sunsolve.sun.com/search/document.do?assetkey=1-21-108993-62-1&searchclause=108993-62 - * http://sunsolve.sun.com/search/document.do?assetkey=1-21-112874-34-1&searchclause=112874-34 - * - * However, many people might not have patched versions, so - * still use our own fix for the buggy version. - */ -#define HAVE_BUGGY_SOLARIS_STRTOD diff --git a/src/include/storage/lmgr.h b/src/include/storage/lmgr.h index 3acc11aa5a3b1..f7cabcbbf550e 100644 --- a/src/include/storage/lmgr.h +++ b/src/include/storage/lmgr.h @@ -59,6 +59,9 @@ extern bool ConditionalLockRelationForExtension(Relation relation, LOCKMODE lockmode); extern int RelationExtensionLockWaiterCount(Relation relation); +/* Lock to recompute pg_database.datfrozenxid in the current database */ +extern void LockDatabaseFrozenIds(LOCKMODE lockmode); + /* Lock a page (currently only used within indexes) */ extern void LockPage(Relation relation, BlockNumber blkno, LOCKMODE lockmode); extern bool ConditionalLockPage(Relation relation, BlockNumber blkno, LOCKMODE lockmode); diff --git a/src/include/storage/lock.h b/src/include/storage/lock.h index fdabf427210ac..1c3e9c1999f56 100644 --- a/src/include/storage/lock.h +++ b/src/include/storage/lock.h @@ -138,6 +138,7 @@ typedef enum LockTagType { LOCKTAG_RELATION, /* whole relation */ LOCKTAG_RELATION_EXTEND, /* the right to extend a relation */ + LOCKTAG_DATABASE_FROZEN_IDS, /* pg_database.datfrozenxid */ LOCKTAG_PAGE, /* one page of a relation */ LOCKTAG_TUPLE, /* one physical tuple */ LOCKTAG_TRANSACTION, /* transaction (for waiting for xact done) */ @@ -194,6 +195,15 @@ typedef struct LOCKTAG (locktag).locktag_type = LOCKTAG_RELATION_EXTEND, \ (locktag).locktag_lockmethodid = DEFAULT_LOCKMETHOD) +/* ID info for frozen IDs is DB OID */ +#define SET_LOCKTAG_DATABASE_FROZEN_IDS(locktag,dboid) \ + ((locktag).locktag_field1 = (dboid), \ + (locktag).locktag_field2 = 0, \ + (locktag).locktag_field3 = 0, \ + (locktag).locktag_field4 = 0, \ + (locktag).locktag_type = LOCKTAG_DATABASE_FROZEN_IDS, \ + (locktag).locktag_lockmethodid = DEFAULT_LOCKMETHOD) + /* ID info for a page is RELATION info + BlockNumber */ #define SET_LOCKTAG_PAGE(locktag,dboid,reloid,blocknum) \ ((locktag).locktag_field1 = (dboid), \ diff --git a/src/test/kerberos/README b/src/test/kerberos/README index 93af72e163679..fa9c03e782915 100644 --- a/src/test/kerberos/README +++ b/src/test/kerberos/README @@ -8,10 +8,12 @@ functionality. This requires a full MIT Kerberos installation, including server and client tools, and is therefore kept separate and not run by default. -Also, this test suite creates a KDC server that listens for TCP/IP -connections on localhost without any real access control, so it is not -safe to run this on a system where there might be untrusted local -users. +CAUTION: The test server run by this test is configured to listen for TCP +connections on localhost. Any user on the same host is able to log in to the +test server while the tests are running. Do not run this suite on a multi-user +system where you don't trust all local users! Also, this test suite creates a +KDC server that listens for TCP/IP connections on localhost without any real +access control. Running the tests ================= diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 8c1b77376fb08..1488bffa2ba34 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -1234,10 +1234,8 @@ sub can_bind return $ret; } -# Automatically shut down any still-running nodes when the test script exits. -# Note that this just stops the postmasters (in the same order the nodes were -# created in). Any temporary directories are deleted, in an unspecified -# order, later when the File::Temp objects are destroyed. +# Automatically shut down any still-running nodes (in the same order the nodes +# were created in) when the test script exits. END { diff --git a/src/test/recovery/t/010_logical_decoding_timelines.pl b/src/test/recovery/t/010_logical_decoding_timelines.pl index 09aaefa9f032e..329500f0ae5b7 100644 --- a/src/test/recovery/t/010_logical_decoding_timelines.pl +++ b/src/test/recovery/t/010_logical_decoding_timelines.pl @@ -111,7 +111,7 @@ # Examine the physical slot the replica uses to stream changes # from the primary to make sure its hot_standby_feedback # has locked in a catalog_xmin on the physical slot, and that -# any xmin is < the catalog_xmin +# any xmin is >= the catalog_xmin $node_primary->poll_query_until( 'postgres', q[ SELECT catalog_xmin IS NOT NULL diff --git a/src/test/ssl/t/SSLServer.pm b/src/test/ssl/t/SSLServer.pm index 1e392b8fbf614..f5987a003efd6 100644 --- a/src/test/ssl/t/SSLServer.pm +++ b/src/test/ssl/t/SSLServer.pm @@ -9,7 +9,6 @@ # - a database called trustdb that lets anyone in # - another database called certdb that uses certificate authentication, ie. # the client must present a valid certificate signed by the client CA -# - two users, called ssltestuser and anotheruser. # # The server is configured to only accept connections from localhost. If you # want to run the client from another host, you'll have to configure that