From: Christoph Berg Date: Tue, 15 Apr 2025 17:11:33 +0000 (+0000) Subject: Read toast chunks in ascending order, looping if required X-Git-Tag: REL_17_2~1 X-Git-Url: http://git.postgresql.org/gitweb/?a=commitdiff_plain;h=3819e05092fc44a416837d682a3c3698a642d1e5;p=pg_filedump.git Read toast chunks in ascending order, looping if required Toast chunks might be stored out-of-order in the table. We previously ignored that and hoped for the best, but our toast.sql test actually exhibits the problem on pre-14 servers where the 5th chunk of the last test is small enough to fit into the first disk block. Fix by looping over the table until all chunks have been read. A smarter solution would require either toast index lookups or caching the chunks. Close #20. Author: Christoph Berg Debugging-By: Svetlana Derevyanko --- diff --git a/decode.c b/decode.c index 69d0d40..168edd4 100644 --- a/decode.c +++ b/decode.c @@ -1364,7 +1364,9 @@ ReadStringFromToast(const char *buffer, varatt_external toast_ptr; char *toast_data = NULL; /* Number of chunks the TOAST data is divided into */ - int32 num_chunks; + uint32 num_chunks; + /* Next chunk to read */ + uint32 want_chunk_id = 0; /* Actual size of external TOASTed value */ int32 toast_ext_size; /* Path to directory with TOAST relation file */ @@ -1372,8 +1374,11 @@ ReadStringFromToast(const char *buffer, /* Filename of TOAST relation file */ char toast_relation_filename[MAXPGPATH]; FILE *toast_rel_fp; + unsigned int toast_relation_block_size; + unsigned int toastDataRead = 0; unsigned int block_options = 0; unsigned int control_options = 0; + int loops = 0; VARATT_EXTERNAL_GET_POINTER(toast_ptr, buffer); @@ -1383,7 +1388,7 @@ ReadStringFromToast(const char *buffer, #else toast_ext_size = toast_ptr.va_extsize; #endif - num_chunks = (toast_ext_size - 1) / TOAST_MAX_CHUNK_SIZE + 1; + num_chunks = ((toast_ext_size - 1) / TOAST_MAX_CHUNK_SIZE) + 1; printf(" TOAST value. Raw size: %8d, external size: %8d, " "value id: %6d, toast relation id: %6d, chunks: %6d\n", @@ -1399,46 +1404,63 @@ ReadStringFromToast(const char *buffer, sprintf(toast_relation_filename, "%s/%d", *toast_relation_path ? toast_relation_path : ".", toast_ptr.va_toastrelid); + free(toast_relation_path); toast_rel_fp = fopen(toast_relation_filename, "rb"); if (!toast_rel_fp) { printf("Cannot open TOAST relation %s\n", toast_relation_filename); - result = -1; + return -1; } - else + + toast_relation_block_size = GetBlockSize(toast_rel_fp); + toast_data = malloc(toast_ptr.va_rawsize); + + /* Loop until all chunks have been read */ + while (want_chunk_id < num_chunks) { - unsigned int toast_relation_block_size = GetBlockSize(toast_rel_fp); + /* Restart at beginning of file */ fseek(toast_rel_fp, 0, SEEK_SET); - toast_data = malloc(toast_ptr.va_rawsize); result = DumpFileContents(block_options, - control_options, - toast_rel_fp, - toast_relation_block_size, - -1, /* no start block */ - -1, /* no end block */ - true, /* is toast relation */ - toast_ptr.va_valueid, - toast_ext_size, - toast_data); - - if (result == 0) - { - if (VARATT_EXTERNAL_IS_COMPRESSED(toast_ptr)) - result = DumpCompressedString(toast_data, toast_ext_size, parse_value); - else - result = parse_value(toast_data, toast_ext_size); - } - else + control_options, + toast_rel_fp, + toast_relation_block_size, + -1, /* no start block */ + -1, /* no end block */ + true, /* is toast relation */ + toast_ptr.va_valueid, + &want_chunk_id, + toast_ext_size, + toast_data, + &toastDataRead); + + if (loops++ > num_chunks) { - printf("Error in TOAST file.\n"); + printf("Not all TOAST chunks found after scanning TOAST table %" PRIu32 " times, giving up\n", + num_chunks); + result = -1; } - free(toast_data); - fclose(toast_rel_fp); + if (result != 0) + break; } - free(toast_relation_path); + fclose(toast_rel_fp); + + if (result == 0) + { + if (VARATT_EXTERNAL_IS_COMPRESSED(toast_ptr)) + result = DumpCompressedString(toast_data, toast_ext_size, parse_value); + else + result = parse_value(toast_data, toast_ext_size); + } + else + { + printf("Error in TOAST file.\n"); + } + + free(toast_data); + } /* If tag is indirect or expanded, it was stored in memory. */ else @@ -1516,6 +1538,7 @@ ToastChunkDecode(const char *tuple_data, Oid toast_oid, Oid *read_toast_oid, uint32 *chunk_id, + uint32 *want_chunk_id, char *chunk_data, unsigned int *chunk_data_size) { @@ -1561,6 +1584,10 @@ ToastChunkDecode(const char *tuple_data, return; } + /* Not the chunk we want to read next */ + if (*chunk_id != *want_chunk_id) + return; + size -= processed_size; data += processed_size; if (size <= 0) @@ -1587,4 +1614,7 @@ ToastChunkDecode(const char *tuple_data, "Partial data: %s\n", size, copyString.data); return; } + + /* advance to next chunk */ + *want_chunk_id += 1; } diff --git a/decode.h b/decode.h index f05f85a..5e0f090 100644 --- a/decode.h +++ b/decode.h @@ -21,6 +21,7 @@ ToastChunkDecode(const char* tuple_data, Oid toast_oid, Oid *read_toast_oid, uint32 *chunk_id, + uint32 *want_chunk_id, char *chunk_data, unsigned int *chunk_data_size); diff --git a/expected/toast_4.out b/expected/toast_4.out index fbdd198..57df04c 100644 --- a/expected/toast_4.out +++ b/expected/toast_4.out @@ -78,7 +78,7 @@ Block 0 ********************************************************
----- Block Offset: 0x00000000 Offsets: Lower 52 (0x0034) Block: Size 8192 Version 4 Upper 7476 (0x1d34) - LSN: logid . recoff 0x........ Special 8192 (0x2000) + LSN: logid ...... recoff 0x........ Special 8192 (0x2000) Items: 7 Free Space: 7424 Checksum: 0x.... Prune XID: 0x00000000 Flags: 0x0004 (ALL_VISIBLE) Length (including item array): 52 diff --git a/pg_filedump.c b/pg_filedump.c index 5b4ec5b..9c6d81e 100644 --- a/pg_filedump.c +++ b/pg_filedump.c @@ -123,6 +123,7 @@ static void FormatBlock(unsigned int blockOptions, unsigned int blockSize, bool isToast, Oid toastOid, + uint32 *want_chunk_id, unsigned int toastExternalSize, char *toastValue, unsigned int *toastRead); @@ -137,6 +138,7 @@ static void FormatItemBlock(char *buffer, Page page, bool isToast, Oid toastOid, + uint32 *want_chunk_id, unsigned int toastExternalSize, char *toastValue, unsigned int *toastRead); @@ -1386,6 +1388,7 @@ FormatItemBlock(char *buffer, Page page, bool isToast, Oid toastOid, + uint32 *want_chunk_id, unsigned int toastExternalSize, char *toastValue, unsigned int *toastRead) @@ -1557,13 +1560,19 @@ FormatItemBlock(char *buffer, Oid read_toast_oid; ToastChunkDecode(&buffer[itemOffset], itemSize, toastOid, - &read_toast_oid, &chunkId, toastValue + *toastRead, + &read_toast_oid, &chunkId, want_chunk_id, toastValue + *toastRead, &chunkSize); if (verbose && read_toast_oid == toastOid) - printf("%s Read TOAST chunk. TOAST Oid: %d, chunk id: %d, " - "chunk data size: %d\n", - indent, toastOid, chunkId, chunkSize); + { + if (chunkId == *want_chunk_id - 1) /* value already incremented */ + printf("%s Read TOAST chunk. TOAST Oid: %d, chunk id: %d, " + "chunk data size: %d\n", + indent, toastOid, chunkId, chunkSize); + else + printf("%s Skipped out-of-order TOAST chunk. TOAST Oid: %d, chunk id: %d\n", + indent, toastOid, chunkId); + } *toastRead += chunkSize; @@ -2096,6 +2105,7 @@ FormatBlock(unsigned int blockOptions, unsigned int blockSize, bool isToast, Oid toastOid, + uint32 *want_chunk_id, unsigned int toastExternalSize, char *toastValue, unsigned int *toastRead) @@ -2132,6 +2142,7 @@ FormatBlock(unsigned int blockOptions, page, isToast, toastOid, + want_chunk_id, toastExternalSize, toastValue, toastRead); @@ -2372,12 +2383,13 @@ DumpFileContents(unsigned int blockOptions, int blockEnd, bool isToast, Oid toastOid, + uint32 *want_chunk_id, unsigned int toastExternalSize, - char *toastValue) + char *toastValue, + unsigned int *toastDataRead) { unsigned int initialRead = 1; unsigned int contentsToDump = 1; - unsigned int toastDataRead = 0; BlockNumber currentBlock = 0; int result = 0; /* On a positive block size, allocate a local buffer to store @@ -2419,7 +2431,7 @@ DumpFileContents(unsigned int blockOptions, * subsequent read gets the error. */ if (initialRead) printf("Error: Premature end of file encountered.\n"); - else if (!(blockOptions & BLOCK_BINARY)) + else if (!(blockOptions & BLOCK_BINARY) && (!isToast || verbose)) printf("\n*** End of File Encountered. Last Block " "Read: %d ***\n", currentBlock - 1); @@ -2445,9 +2457,10 @@ DumpFileContents(unsigned int blockOptions, blockSize, isToast, toastOid, + want_chunk_id, toastExternalSize, toastValue, - &toastDataRead); + toastDataRead); } } } @@ -2468,7 +2481,7 @@ DumpFileContents(unsigned int blockOptions, initialRead = 0; /* If TOAST data is read */ - if (isToast && toastDataRead >= toastExternalSize) + if (isToast && *toastDataRead >= toastExternalSize) break; } @@ -2583,8 +2596,10 @@ main(int argv, char **argc) blockEnd, false /* is toast realtion */, 0, /* no toast Oid */ + NULL, /* no current TOAST chunk */ 0, /* no toast external size */ - NULL /* no out toast value */ + NULL, /* no out toast value */ + NULL /* no toast value length */ ); } diff --git a/pg_filedump.h b/pg_filedump.h index a836f01..c552134 100644 --- a/pg_filedump.h +++ b/pg_filedump.h @@ -154,4 +154,4 @@ unsigned int GetBlockSize(FILE *fp); int DumpFileContents(unsigned int blockOptions, unsigned int controlOptions, FILE *fp, unsigned int blockSize, int blockStart, int blockEnd, bool isToast, Oid toastOid, - unsigned int toastExternalSize, char *toastValue); + uint32 *want_chunk_id, unsigned int toastExternalSize, char *toastValue, unsigned int *toastDataRead);