line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
/* |
2
|
|
|
|
|
|
|
* Routines for matching and writing files in the pool. |
3
|
|
|
|
|
|
|
* |
4
|
|
|
|
|
|
|
* Copyright (C) 2013 Craig Barratt. |
5
|
|
|
|
|
|
|
* |
6
|
|
|
|
|
|
|
* This program is free software; you can redistribute it and/or modify |
7
|
|
|
|
|
|
|
* it under the terms of the GNU General Public License as published by |
8
|
|
|
|
|
|
|
* the Free Software Foundation; either version 3 of the License, or |
9
|
|
|
|
|
|
|
* (at your option) any later version. |
10
|
|
|
|
|
|
|
* |
11
|
|
|
|
|
|
|
* This program is distributed in the hope that it will be useful, |
12
|
|
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
13
|
|
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
14
|
|
|
|
|
|
|
* GNU General Public License for more details. |
15
|
|
|
|
|
|
|
* |
16
|
|
|
|
|
|
|
* You should have received a copy of the GNU General Public License along |
17
|
|
|
|
|
|
|
* with this program; if not, visit the http://fsf.org website. |
18
|
|
|
|
|
|
|
*/ |
19
|
|
|
|
|
|
|
|
20
|
|
|
|
|
|
|
#include "backuppc.h" |
21
|
|
|
|
|
|
|
|
22
|
|
|
|
|
|
|
|
23
|
|
|
|
|
|
|
static uint32 PoolWriteCnt = 0; |
24
|
|
|
|
|
|
|
|
25
|
|
|
|
|
|
|
/* |
26
|
|
|
|
|
|
|
* Buffer used in various places for copying, comparing etc |
27
|
|
|
|
|
|
|
*/ |
28
|
|
|
|
|
|
|
#define COMPARE_BUF_SZ (1 << 20) /* 1.0 MB */ |
29
|
|
|
|
|
|
|
static uchar TempBuf[2 * COMPARE_BUF_SZ]; |
30
|
|
|
|
|
|
|
|
31
|
|
|
|
|
|
|
/* |
32
|
|
|
|
|
|
|
* A freelist of unused BPC_POOL_WRITE_BUF_SZ sized-buffers. |
33
|
|
|
|
|
|
|
* We use the first sizeof(void*) bytes of the buffer as a single-linked |
34
|
|
|
|
|
|
|
* list, with a NULL at the end. |
35
|
|
|
|
|
|
|
*/ |
36
|
|
|
|
|
|
|
static void *DataBufferFreeList = (void*)NULL; |
37
|
|
|
|
|
|
|
|
38
|
0
|
|
|
|
|
|
int bpc_poolWrite_open(bpc_poolWrite_info *info, int compress, bpc_digest *digest) |
39
|
|
|
|
|
|
|
{ |
40
|
|
|
|
|
|
|
int i; |
41
|
|
|
|
|
|
|
|
42
|
0
|
|
|
|
|
|
info->compress = compress; |
43
|
0
|
|
|
|
|
|
info->eof = 0; |
44
|
0
|
|
|
|
|
|
info->errorCnt = 0; |
45
|
0
|
|
|
|
|
|
info->state = 0; |
46
|
0
|
|
|
|
|
|
info->bufferIdx = 0; |
47
|
0
|
|
|
|
|
|
info->fileSize = 0; |
48
|
0
|
|
|
|
|
|
info->matchPosn = 0; |
49
|
0
|
|
|
|
|
|
info->candidateList = NULL; |
50
|
0
|
|
|
|
|
|
info->fdOpen = 0; |
51
|
0
|
|
|
|
|
|
info->retValue = -1; |
52
|
0
|
|
|
|
|
|
info->poolFileSize = 0; |
53
|
0
|
|
|
|
|
|
info->retryCnt = 0; |
54
|
0
|
|
|
|
|
|
info->digestExtOpen = -1; |
55
|
0
|
|
|
|
|
|
info->digestExtZeroLen = -1; |
56
|
0
|
0
|
|
|
|
|
for ( i = 0 ; i < BPC_POOL_WRITE_CONCURRENT_MATCH ; i++ ) { |
57
|
0
|
|
|
|
|
|
info->match[i].used = 0; |
58
|
|
|
|
|
|
|
} |
59
|
0
|
0
|
|
|
|
|
if ( DataBufferFreeList ) { |
60
|
0
|
|
|
|
|
|
info->buffer = DataBufferFreeList; |
61
|
0
|
|
|
|
|
|
DataBufferFreeList = *(void**)DataBufferFreeList; |
62
|
|
|
|
|
|
|
} else { |
63
|
0
|
|
|
|
|
|
info->buffer = malloc(BPC_POOL_WRITE_BUF_SZ); |
64
|
|
|
|
|
|
|
} |
65
|
0
|
0
|
|
|
|
|
if ( !info->buffer ) { |
66
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_open: can't allocate %d bytes for buffer\n", BPC_POOL_WRITE_BUF_SZ); |
67
|
0
|
|
|
|
|
|
return -1; |
68
|
|
|
|
|
|
|
} |
69
|
0
|
0
|
|
|
|
|
if ( digest ) { |
70
|
0
|
|
|
|
|
|
info->digest = *digest; |
71
|
|
|
|
|
|
|
/* TODO: don't have V3 digest at this point! */ |
72
|
0
|
|
|
|
|
|
info->state = 2; |
73
|
|
|
|
|
|
|
} else { |
74
|
0
|
|
|
|
|
|
info->digest.len = 0; |
75
|
|
|
|
|
|
|
} |
76
|
0
|
|
|
|
|
|
info->digest_v3.len = 0; |
77
|
0
|
0
|
|
|
|
|
if ( snprintf(info->tmpFileName, sizeof(info->tmpFileName), "%s/%d.%d.%d", |
|
|
0
|
|
|
|
|
|
78
|
0
|
|
|
|
|
|
compress ? BPC_CPoolDir : BPC_PoolDir, (int)getpid(), PoolWriteCnt++, |
79
|
|
|
|
|
|
|
BPC_TmpFileUnique >= 0 ? BPC_TmpFileUnique : 0) >= (int)sizeof(info->tmpFileName) - 1 ) { |
80
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_open: file name too long %s\n", info->tmpFileName); |
81
|
0
|
|
|
|
|
|
return -1; |
82
|
|
|
|
|
|
|
} |
83
|
0
|
|
|
|
|
|
return 0; |
84
|
|
|
|
|
|
|
} |
85
|
|
|
|
|
|
|
|
86
|
|
|
|
|
|
|
/* |
87
|
|
|
|
|
|
|
* Fill out the array of candidate matching files. Returns the number of active |
88
|
|
|
|
|
|
|
* matching files. |
89
|
|
|
|
|
|
|
*/ |
90
|
0
|
|
|
|
|
|
static int bpc_poolWrite_updateMatches(bpc_poolWrite_info *info) |
91
|
|
|
|
|
|
|
{ |
92
|
0
|
|
|
|
|
|
int i, nMatch = 0; |
93
|
|
|
|
|
|
|
|
94
|
0
|
0
|
|
|
|
|
for ( i = 0 ; i < BPC_POOL_WRITE_CONCURRENT_MATCH ; i++ ) { |
95
|
0
|
0
|
|
|
|
|
if ( info->match[i].used ) { |
96
|
0
|
|
|
|
|
|
nMatch++; |
97
|
0
|
|
|
|
|
|
continue; |
98
|
|
|
|
|
|
|
} |
99
|
0
|
0
|
|
|
|
|
while ( info->candidateList ) { |
100
|
0
|
|
|
|
|
|
int match = 1; |
101
|
|
|
|
|
|
|
bpc_candidate_file *candidateFile; |
102
|
|
|
|
|
|
|
|
103
|
0
|
|
|
|
|
|
candidateFile = info->candidateList; |
104
|
0
|
|
|
|
|
|
info->candidateList = candidateFile->next; |
105
|
0
|
0
|
|
|
|
|
if ( bpc_fileZIO_open(&info->match[i].fd, candidateFile->fileName, 0, info->compress) ) { |
106
|
0
|
|
|
|
|
|
info->errorCnt++; |
107
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_updateMatches: can't open candidate file %s for read\n", |
108
|
0
|
|
|
|
|
|
candidateFile->fileName); |
109
|
0
|
|
|
|
|
|
free(candidateFile); |
110
|
0
|
|
|
|
|
|
continue; |
111
|
|
|
|
|
|
|
} |
112
|
|
|
|
|
|
|
/* |
113
|
|
|
|
|
|
|
* We need to check that the first info->matchPosn bytes of the candidate file match |
114
|
|
|
|
|
|
|
* the original file. |
115
|
|
|
|
|
|
|
*/ |
116
|
0
|
0
|
|
|
|
|
if ( info->matchPosn > 0 ) { |
117
|
0
|
0
|
|
|
|
|
if ( info->fdOpen ) { |
118
|
|
|
|
|
|
|
/* |
119
|
|
|
|
|
|
|
* Compare the candidate file against the data in the file |
120
|
|
|
|
|
|
|
*/ |
121
|
0
|
|
|
|
|
|
uchar *buf0 = TempBuf; |
122
|
0
|
|
|
|
|
|
uchar *buf1 = TempBuf + COMPARE_BUF_SZ; |
123
|
0
|
|
|
|
|
|
OFF_T idx = 0; |
124
|
|
|
|
|
|
|
|
125
|
0
|
|
|
|
|
|
bpc_fileZIO_rewind(&info->fd); |
126
|
0
|
0
|
|
|
|
|
while ( idx < info->matchPosn ) { |
127
|
0
|
|
|
|
|
|
OFF_T thisRead = info->matchPosn - idx; |
128
|
|
|
|
|
|
|
OFF_T nread0, nread1; |
129
|
|
|
|
|
|
|
|
130
|
0
|
0
|
|
|
|
|
if ( thisRead > COMPARE_BUF_SZ ) thisRead = COMPARE_BUF_SZ; |
131
|
0
|
|
|
|
|
|
nread0 = bpc_fileZIO_read(&info->fd, buf0, thisRead); |
132
|
0
|
|
|
|
|
|
nread1 = bpc_fileZIO_read(&info->match[i].fd, buf1, thisRead); |
133
|
0
|
0
|
|
|
|
|
if ( nread0 != nread1 || memcmp(buf0, buf1, nread0) ) { |
|
|
0
|
|
|
|
|
|
134
|
|
|
|
|
|
|
/* |
135
|
|
|
|
|
|
|
* Need to keep reading the original file to get back to matchPosn |
136
|
|
|
|
|
|
|
*/ |
137
|
0
|
|
|
|
|
|
match = 0; |
138
|
|
|
|
|
|
|
} |
139
|
0
|
|
|
|
|
|
idx += nread0; |
140
|
|
|
|
|
|
|
} |
141
|
|
|
|
|
|
|
} else { |
142
|
|
|
|
|
|
|
/* |
143
|
|
|
|
|
|
|
* Compare the candidate file against the data in the buffer |
144
|
|
|
|
|
|
|
*/ |
145
|
0
|
|
|
|
|
|
uchar *buf1 = TempBuf; |
146
|
0
|
|
|
|
|
|
OFF_T idx = 0; |
147
|
|
|
|
|
|
|
|
148
|
0
|
0
|
|
|
|
|
while ( idx < info->matchPosn ) { |
149
|
0
|
|
|
|
|
|
OFF_T thisRead = info->matchPosn - idx; |
150
|
|
|
|
|
|
|
OFF_T nread1; |
151
|
|
|
|
|
|
|
|
152
|
0
|
0
|
|
|
|
|
if ( thisRead > COMPARE_BUF_SZ ) thisRead = COMPARE_BUF_SZ; |
153
|
0
|
0
|
|
|
|
|
if ( thisRead > info->bufferIdx - idx ) thisRead = info->bufferIdx - idx; |
154
|
0
|
|
|
|
|
|
nread1 = bpc_fileZIO_read(&info->match[i].fd, buf1, thisRead); |
155
|
0
|
0
|
|
|
|
|
if ( thisRead != nread1 || memcmp(info->buffer + idx, buf1, thisRead) ) { |
|
|
0
|
|
|
|
|
|
156
|
0
|
|
|
|
|
|
match = 0; |
157
|
0
|
|
|
|
|
|
break; |
158
|
|
|
|
|
|
|
} |
159
|
0
|
|
|
|
|
|
idx += thisRead; |
160
|
|
|
|
|
|
|
} |
161
|
|
|
|
|
|
|
} |
162
|
|
|
|
|
|
|
} |
163
|
0
|
0
|
|
|
|
|
if ( !match ) { |
164
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 8 ) bpc_logMsgf("Discarding %s since it doesn't match starting portion\n", candidateFile->fileName); |
165
|
0
|
|
|
|
|
|
bpc_fileZIO_close(&info->match[i].fd); |
166
|
0
|
|
|
|
|
|
free(candidateFile); |
167
|
0
|
|
|
|
|
|
continue; |
168
|
|
|
|
|
|
|
} |
169
|
0
|
|
|
|
|
|
info->match[i].used = 1; |
170
|
0
|
|
|
|
|
|
info->match[i].digest = candidateFile->digest; |
171
|
0
|
|
|
|
|
|
info->match[i].v3File = candidateFile->v3File; |
172
|
0
|
|
|
|
|
|
info->match[i].fileSize = candidateFile->fileSize; |
173
|
0
|
|
|
|
|
|
strcpy(info->match[i].fileName, candidateFile->fileName); |
174
|
0
|
|
|
|
|
|
nMatch++; |
175
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 9 ) bpc_logMsgf("match[%d] now set to %s\n", i, info->match[i].fileName); |
176
|
0
|
|
|
|
|
|
free(candidateFile); |
177
|
0
|
|
|
|
|
|
break; |
178
|
|
|
|
|
|
|
} |
179
|
|
|
|
|
|
|
} |
180
|
0
|
|
|
|
|
|
return nMatch; |
181
|
|
|
|
|
|
|
} |
182
|
|
|
|
|
|
|
|
183
|
|
|
|
|
|
|
/* |
184
|
|
|
|
|
|
|
* Write a chunk to the current pool file. |
185
|
|
|
|
|
|
|
* |
186
|
|
|
|
|
|
|
* Call with undef to indicate EOF / close. |
187
|
|
|
|
|
|
|
*/ |
188
|
0
|
|
|
|
|
|
int bpc_poolWrite_write(bpc_poolWrite_info *info, uchar *data, size_t dataLen) |
189
|
|
|
|
|
|
|
{ |
190
|
0
|
0
|
|
|
|
|
if ( info->errorCnt ) return -1; |
191
|
|
|
|
|
|
|
|
192
|
0
|
|
|
|
|
|
info->fileSize += dataLen; |
193
|
|
|
|
|
|
|
|
194
|
0
|
0
|
|
|
|
|
if ( info->state == 0 ) { |
195
|
|
|
|
|
|
|
/* |
196
|
|
|
|
|
|
|
* In this state we are at the start of the file and don't have a digest yet |
197
|
|
|
|
|
|
|
*/ |
198
|
0
|
0
|
|
|
|
|
if ( data ) { |
199
|
|
|
|
|
|
|
/* |
200
|
|
|
|
|
|
|
* Cumulate small writes at the start of the file |
201
|
|
|
|
|
|
|
*/ |
202
|
0
|
0
|
|
|
|
|
if ( info->bufferIdx + dataLen <= BPC_POOL_WRITE_BUF_SZ ) { |
203
|
0
|
|
|
|
|
|
memcpy(info->buffer + info->bufferIdx, data, dataLen); |
204
|
0
|
|
|
|
|
|
info->bufferIdx += dataLen; |
205
|
0
|
|
|
|
|
|
return 0; |
206
|
|
|
|
|
|
|
} |
207
|
|
|
|
|
|
|
|
208
|
|
|
|
|
|
|
/* |
209
|
|
|
|
|
|
|
* We have more data than the buffer can fit. Top off the buffer if it has less than |
210
|
|
|
|
|
|
|
* 1MB of data so that we can compute the V3 digest. |
211
|
|
|
|
|
|
|
*/ |
212
|
0
|
0
|
|
|
|
|
if ( data && info->bufferIdx < (1 << 20) && BPC_POOL_WRITE_BUF_SZ >= (1 << 20) ) { |
|
|
0
|
|
|
|
|
|
213
|
0
|
|
|
|
|
|
uint32 addTo1MB = (1 << 20) - info->bufferIdx; |
214
|
0
|
|
|
|
|
|
memcpy(info->buffer + info->bufferIdx, data, addTo1MB); |
215
|
0
|
|
|
|
|
|
info->bufferIdx += addTo1MB; |
216
|
0
|
|
|
|
|
|
data += addTo1MB; |
217
|
0
|
|
|
|
|
|
dataLen -= addTo1MB; |
218
|
|
|
|
|
|
|
} |
219
|
|
|
|
|
|
|
|
220
|
0
|
0
|
|
|
|
|
if ( !info->digest.len ) { |
221
|
|
|
|
|
|
|
ssize_t writeRet; |
222
|
|
|
|
|
|
|
/* |
223
|
|
|
|
|
|
|
* We don't have a digest and the file is bigger than the buffer. |
224
|
|
|
|
|
|
|
* So we need to write the data to a temp file and compute the MD5 |
225
|
|
|
|
|
|
|
* digest as we write the file. |
226
|
|
|
|
|
|
|
*/ |
227
|
0
|
0
|
|
|
|
|
if ( bpc_fileZIO_open(&info->fd, info->tmpFileName, 1, info->compress) ) { |
228
|
0
|
|
|
|
|
|
info->errorCnt++; |
229
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_write: can't open/create %s for writing", info->tmpFileName); |
230
|
0
|
|
|
|
|
|
return -1; |
231
|
|
|
|
|
|
|
} |
232
|
0
|
|
|
|
|
|
info->fdOpen = 1; |
233
|
0
|
|
|
|
|
|
md5_begin(&info->md5); |
234
|
0
|
0
|
|
|
|
|
if ( info->bufferIdx > 0 ) { |
235
|
0
|
0
|
|
|
|
|
if ( (writeRet = bpc_fileZIO_write(&info->fd, info->buffer, info->bufferIdx)) != (signed)info->bufferIdx ) { |
236
|
0
|
|
|
|
|
|
info->errorCnt++; |
237
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_write: write of %lu bytes to %s failed, return = %d", |
238
|
0
|
|
|
|
|
|
(unsigned long)info->bufferIdx, info->tmpFileName, (int)writeRet); |
239
|
0
|
|
|
|
|
|
return -1; |
240
|
|
|
|
|
|
|
} |
241
|
0
|
|
|
|
|
|
md5_update(&info->md5, info->buffer, info->bufferIdx); |
242
|
|
|
|
|
|
|
} |
243
|
0
|
|
|
|
|
|
info->state = 1; |
244
|
|
|
|
|
|
|
} else { |
245
|
|
|
|
|
|
|
/* |
246
|
|
|
|
|
|
|
* We have the new digest, so figure out the list of candidate matching files |
247
|
|
|
|
|
|
|
*/ |
248
|
|
|
|
|
|
|
/* TODO: don't have V3 digest at this point! */ |
249
|
0
|
|
|
|
|
|
info->state = 2; |
250
|
|
|
|
|
|
|
} |
251
|
|
|
|
|
|
|
} else { |
252
|
|
|
|
|
|
|
/* |
253
|
|
|
|
|
|
|
* We are at EOF, so we can compute the digests based on the entire file in |
254
|
|
|
|
|
|
|
* the buffer. |
255
|
|
|
|
|
|
|
*/ |
256
|
0
|
|
|
|
|
|
info->eof = 1; |
257
|
0
|
|
|
|
|
|
bpc_digest_buffer2MD5(&info->digest, info->buffer, info->bufferIdx); |
258
|
0
|
0
|
|
|
|
|
if ( BPC_PoolV3Enabled ) { |
259
|
0
|
|
|
|
|
|
bpc_digest_buffer2MD5_v3(&info->digest_v3, info->buffer, info->bufferIdx); |
260
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 8 ) { |
261
|
|
|
|
|
|
|
char hexStr_v3[BPC_DIGEST_LEN_MAX * 2 + 1], hexStr[BPC_DIGEST_LEN_MAX * 2 + 1]; |
262
|
0
|
|
|
|
|
|
bpc_digest_digest2str(&info->digest, hexStr); |
263
|
0
|
|
|
|
|
|
bpc_digest_digest2str(&info->digest_v3, hexStr_v3); |
264
|
0
|
|
|
|
|
|
bpc_logMsgf("bpc_poolWrite_write: digest is %s, v3 is %s\n", hexStr, hexStr_v3); |
265
|
|
|
|
|
|
|
} |
266
|
0
|
0
|
|
|
|
|
} else if ( BPC_LogLevel >= 8 ) { |
267
|
|
|
|
|
|
|
char hexStr[BPC_DIGEST_LEN_MAX * 2 + 1]; |
268
|
0
|
|
|
|
|
|
bpc_digest_digest2str(&info->digest, hexStr); |
269
|
0
|
|
|
|
|
|
bpc_logMsgf("bpc_poolWrite_write: digest is %s\n", hexStr); |
270
|
|
|
|
|
|
|
} |
271
|
0
|
|
|
|
|
|
info->state = 2; |
272
|
|
|
|
|
|
|
} |
273
|
|
|
|
|
|
|
} |
274
|
0
|
0
|
|
|
|
|
if ( info->state == 1 ) { |
275
|
|
|
|
|
|
|
ssize_t writeRet; |
276
|
|
|
|
|
|
|
/* |
277
|
|
|
|
|
|
|
* In this state we are writing the data to a compressed temporary file, and |
278
|
|
|
|
|
|
|
* accumulating the digests. |
279
|
|
|
|
|
|
|
*/ |
280
|
0
|
0
|
|
|
|
|
if ( dataLen > 0 ) { |
281
|
0
|
0
|
|
|
|
|
if ( (writeRet = bpc_fileZIO_write(&info->fd, data, dataLen)) != (ssize_t)dataLen ) { |
282
|
0
|
|
|
|
|
|
info->errorCnt++; |
283
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_write: write of %lu bytes to %s failed, return = %d", |
284
|
0
|
|
|
|
|
|
(unsigned long)dataLen, info->tmpFileName, (int)writeRet); |
285
|
0
|
|
|
|
|
|
return -1; |
286
|
|
|
|
|
|
|
} |
287
|
0
|
|
|
|
|
|
md5_update(&info->md5, data, dataLen); |
288
|
|
|
|
|
|
|
} |
289
|
0
|
0
|
|
|
|
|
if ( !data ) { |
290
|
|
|
|
|
|
|
/* |
291
|
|
|
|
|
|
|
* We are at EOF. Close the output file and re-open it for reading. |
292
|
|
|
|
|
|
|
* Compute the digests too. |
293
|
|
|
|
|
|
|
*/ |
294
|
0
|
|
|
|
|
|
bpc_fileZIO_close(&info->fd); |
295
|
0
|
0
|
|
|
|
|
if ( bpc_fileZIO_open(&info->fd, info->tmpFileName, 0, info->compress) ) { |
296
|
0
|
|
|
|
|
|
info->errorCnt++; |
297
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_write: can't open %s for reading", info->tmpFileName); |
298
|
0
|
|
|
|
|
|
return -1; |
299
|
|
|
|
|
|
|
} |
300
|
0
|
|
|
|
|
|
info->fdOpen = 1; |
301
|
0
|
|
|
|
|
|
md5_result(&info->md5, info->digest.digest); |
302
|
0
|
|
|
|
|
|
info->digest.len = MD5_DIGEST_LEN; |
303
|
0
|
0
|
|
|
|
|
if ( BPC_PoolV3Enabled ) { |
304
|
0
|
|
|
|
|
|
bpc_digest_buffer2MD5_v3(&info->digest_v3, info->buffer, info->fileSize); |
305
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 8 ) { |
306
|
|
|
|
|
|
|
char hexStr_v3[BPC_DIGEST_LEN_MAX * 2 + 1], hexStr[BPC_DIGEST_LEN_MAX * 2 + 1]; |
307
|
0
|
|
|
|
|
|
bpc_digest_digest2str(&info->digest, hexStr); |
308
|
0
|
|
|
|
|
|
bpc_digest_digest2str(&info->digest_v3, hexStr_v3); |
309
|
0
|
|
|
|
|
|
bpc_logMsgf("bpc_poolWrite_write: digest is %s, v3 is %s\n", hexStr, hexStr_v3); |
310
|
|
|
|
|
|
|
} |
311
|
0
|
0
|
|
|
|
|
} else if ( BPC_LogLevel >= 8 ) { |
312
|
|
|
|
|
|
|
char hexStr[BPC_DIGEST_LEN_MAX * 2 + 1]; |
313
|
0
|
|
|
|
|
|
bpc_digest_digest2str(&info->digest, hexStr); |
314
|
0
|
|
|
|
|
|
bpc_logMsgf("bpc_poolWrite_write: digest is %s\n", hexStr); |
315
|
|
|
|
|
|
|
} |
316
|
0
|
|
|
|
|
|
info->state = 2; |
317
|
|
|
|
|
|
|
} |
318
|
|
|
|
|
|
|
} |
319
|
0
|
0
|
|
|
|
|
if ( info->state == 2 ) { |
320
|
0
|
|
|
|
|
|
uint32 ext = 0; |
321
|
|
|
|
|
|
|
char poolPath[BPC_MAXPATHLEN]; |
322
|
|
|
|
|
|
|
STRUCT_STAT st; |
323
|
|
|
|
|
|
|
|
324
|
|
|
|
|
|
|
/* |
325
|
|
|
|
|
|
|
* In this state we have either the full file in info->buffer, or the full file |
326
|
|
|
|
|
|
|
* is opened for reading with info->fd. We also have digests computed. |
327
|
|
|
|
|
|
|
* |
328
|
|
|
|
|
|
|
* We figure out the list of candidate files to match. If there are any |
329
|
|
|
|
|
|
|
* new digest files then we just try to match them. Otherwise we also |
330
|
|
|
|
|
|
|
* try to match any old V3 files. |
331
|
|
|
|
|
|
|
* |
332
|
|
|
|
|
|
|
* Since the empty file is never stored in the pool, we have to make sure |
333
|
|
|
|
|
|
|
* that any digest that collides (ie: 0xd41d8cd98f00b204e9800998ecf8427e) |
334
|
|
|
|
|
|
|
* doesn't use the first slot (ie: make sure it has an extension > 0) |
335
|
|
|
|
|
|
|
*/ |
336
|
|
|
|
|
|
|
static uchar zeroLenMD5[] = { |
337
|
|
|
|
|
|
|
0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e |
338
|
|
|
|
|
|
|
}; |
339
|
0
|
0
|
|
|
|
|
if ( info->fileSize > 0 && !memcmp(info->digest.digest, zeroLenMD5, sizeof(zeroLenMD5)) ) { |
|
|
0
|
|
|
|
|
|
340
|
0
|
|
|
|
|
|
ext++; |
341
|
|
|
|
|
|
|
} |
342
|
|
|
|
|
|
|
|
343
|
0
|
|
|
|
|
|
info->digestExtZeroLen = -1; |
344
|
|
|
|
|
|
|
while ( 1 ) { |
345
|
0
|
|
|
|
|
|
bpc_digest_append_ext(&info->digest, ext); |
346
|
0
|
|
|
|
|
|
bpc_digest_md52path(poolPath, info->compress, &info->digest); |
347
|
|
|
|
|
|
|
/* |
348
|
|
|
|
|
|
|
* For >= V4.x pool, don't attempt to match pool files that |
349
|
|
|
|
|
|
|
* are empty, since in >= V4.x we don't rename pool |
350
|
|
|
|
|
|
|
* files in a repeated chain and instead replace them |
351
|
|
|
|
|
|
|
* with an empty file. |
352
|
|
|
|
|
|
|
* If the candidate has the other execute bit set, we do a safe |
353
|
|
|
|
|
|
|
* reset of the bit and allow matches to occur. This is used to flag |
354
|
|
|
|
|
|
|
* pool files that will be deleted next time BackupPC_refCountUpdate |
355
|
|
|
|
|
|
|
* runs, so resetting that bit prevents the deletion. |
356
|
|
|
|
|
|
|
*/ |
357
|
0
|
0
|
|
|
|
|
if ( stat(poolPath, &st) ) break; |
358
|
0
|
0
|
|
|
|
|
if ( S_ISREG(st.st_mode) ) { |
359
|
0
|
0
|
|
|
|
|
if ( st.st_size > 0 ) { |
360
|
|
|
|
|
|
|
bpc_candidate_file *candidateFile; |
361
|
0
|
0
|
|
|
|
|
if ( (st.st_mode & S_IXOTH) && bpc_poolWrite_unmarkPendingDelete(poolPath) ) { |
|
|
0
|
|
|
|
|
|
362
|
0
|
|
|
|
|
|
bpc_logErrf("Couldn't unmark candidate matching file %s (skipped; errno = %d)\n", poolPath, errno); |
363
|
0
|
|
|
|
|
|
info->errorCnt++; |
364
|
0
|
|
|
|
|
|
break; |
365
|
|
|
|
|
|
|
} |
366
|
0
|
|
|
|
|
|
candidateFile = malloc(sizeof(bpc_candidate_file)); |
367
|
0
|
0
|
|
|
|
|
if ( !candidateFile ) { |
368
|
0
|
|
|
|
|
|
info->errorCnt++; |
369
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_write: can't allocate bpc_candidate_file\n"); |
370
|
0
|
|
|
|
|
|
return -1; |
371
|
|
|
|
|
|
|
} |
372
|
0
|
|
|
|
|
|
candidateFile->digest = info->digest; |
373
|
0
|
|
|
|
|
|
candidateFile->fileSize = st.st_size; |
374
|
0
|
|
|
|
|
|
candidateFile->v3File = 0; |
375
|
0
|
|
|
|
|
|
strcpy(candidateFile->fileName, poolPath); |
376
|
0
|
|
|
|
|
|
candidateFile->next = info->candidateList; |
377
|
0
|
|
|
|
|
|
info->candidateList = candidateFile; |
378
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 7 ) bpc_logMsgf("Candidate matching file %s\n", candidateFile->fileName); |
379
|
0
|
0
|
|
|
|
|
} else if ( info->digestExtZeroLen < 0 ) { |
380
|
|
|
|
|
|
|
/* |
381
|
|
|
|
|
|
|
* Remember the first empty file in case we have to insert a |
382
|
|
|
|
|
|
|
* new pool file here. |
383
|
|
|
|
|
|
|
*/ |
384
|
0
|
|
|
|
|
|
info->digestExtZeroLen = ext; |
385
|
|
|
|
|
|
|
} |
386
|
|
|
|
|
|
|
} |
387
|
0
|
|
|
|
|
|
ext++; |
388
|
0
|
|
|
|
|
|
} |
389
|
|
|
|
|
|
|
/* |
390
|
|
|
|
|
|
|
* Remember the next open slot in case we have to add a new pool |
391
|
|
|
|
|
|
|
* file here. |
392
|
|
|
|
|
|
|
*/ |
393
|
0
|
|
|
|
|
|
info->digestExtOpen = ext; |
394
|
0
|
|
|
|
|
|
bpc_digest_append_ext(&info->digest, 0); |
395
|
|
|
|
|
|
|
|
396
|
0
|
0
|
|
|
|
|
if ( BPC_PoolV3Enabled && !info->candidateList ) { |
|
|
0
|
|
|
|
|
|
397
|
|
|
|
|
|
|
/* |
398
|
|
|
|
|
|
|
* No matching candidate files so far, so now look in V3 pool |
399
|
|
|
|
|
|
|
*/ |
400
|
0
|
|
|
|
|
|
ext = 0; |
401
|
|
|
|
|
|
|
while ( 1 ) { |
402
|
0
|
|
|
|
|
|
bpc_digest_append_ext(&info->digest_v3, ext); |
403
|
0
|
|
|
|
|
|
bpc_digest_md52path_v3(poolPath, info->compress, &info->digest_v3); |
404
|
0
|
|
|
|
|
|
ext++; |
405
|
|
|
|
|
|
|
|
406
|
|
|
|
|
|
|
/* |
407
|
|
|
|
|
|
|
* For V3.x pool, don't attempt to match pool files: |
408
|
|
|
|
|
|
|
* - that already have too many hardlinks. |
409
|
|
|
|
|
|
|
* - with only one link since starting in BackupPC v3.0, |
410
|
|
|
|
|
|
|
* BackupPC_nightly could be running in parallel (and |
411
|
|
|
|
|
|
|
* removing those files). This doesn't eliminate all |
412
|
|
|
|
|
|
|
* possible race conditions, but just reduces the |
413
|
|
|
|
|
|
|
* odds. Other design steps eliminate the remaining |
414
|
|
|
|
|
|
|
* race conditions of linking vs removing. |
415
|
|
|
|
|
|
|
*/ |
416
|
0
|
0
|
|
|
|
|
if ( stat(poolPath, &st) ) break; |
417
|
0
|
0
|
|
|
|
|
if ( S_ISREG(st.st_mode) |
418
|
0
|
0
|
|
|
|
|
&& 1 < st.st_nlink && st.st_nlink < (unsigned)BPC_HardLinkMax ) { |
|
|
0
|
|
|
|
|
|
419
|
0
|
|
|
|
|
|
bpc_candidate_file *candidateFile = malloc(sizeof(bpc_candidate_file)); |
420
|
0
|
0
|
|
|
|
|
if ( !candidateFile ) { |
421
|
0
|
|
|
|
|
|
info->errorCnt++; |
422
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_write: can't allocate bpc_candidate_file\n"); |
423
|
0
|
|
|
|
|
|
return -1; |
424
|
|
|
|
|
|
|
} |
425
|
0
|
|
|
|
|
|
candidateFile->digest = info->digest_v3; |
426
|
0
|
|
|
|
|
|
candidateFile->fileSize = st.st_size; |
427
|
0
|
|
|
|
|
|
candidateFile->v3File = 1; |
428
|
0
|
|
|
|
|
|
strcpy(candidateFile->fileName, poolPath); |
429
|
0
|
|
|
|
|
|
candidateFile->next = info->candidateList; |
430
|
0
|
|
|
|
|
|
info->candidateList = candidateFile; |
431
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 7 ) bpc_logMsgf("Candidate v3 matching file %s\n", candidateFile->fileName); |
432
|
|
|
|
|
|
|
} |
433
|
0
|
|
|
|
|
|
} |
434
|
0
|
|
|
|
|
|
bpc_digest_append_ext(&info->digest_v3, 0); |
435
|
|
|
|
|
|
|
} |
436
|
|
|
|
|
|
|
/* |
437
|
|
|
|
|
|
|
* Open the first set of candidate files. |
438
|
|
|
|
|
|
|
*/ |
439
|
0
|
|
|
|
|
|
bpc_poolWrite_updateMatches(info); |
440
|
0
|
|
|
|
|
|
info->state = 3; |
441
|
|
|
|
|
|
|
} |
442
|
0
|
0
|
|
|
|
|
if ( info->state == 3 ) { |
443
|
|
|
|
|
|
|
/* |
444
|
|
|
|
|
|
|
* In this state we are continuing to match against candidate files |
445
|
|
|
|
|
|
|
*/ |
446
|
|
|
|
|
|
|
while ( 1 ) { |
447
|
0
|
|
|
|
|
|
int i, replaceCnt = 0, nMatch = 0; |
448
|
0
|
|
|
|
|
|
uchar *buf0 = TempBuf; |
449
|
0
|
|
|
|
|
|
uchar *buf1 = TempBuf + COMPARE_BUF_SZ; |
450
|
|
|
|
|
|
|
uchar *buf; |
451
|
|
|
|
|
|
|
OFF_T nread0; |
452
|
|
|
|
|
|
|
|
453
|
0
|
0
|
|
|
|
|
if ( info->fdOpen ) { |
454
|
0
|
|
|
|
|
|
nread0 = bpc_fileZIO_read(&info->fd, buf0, COMPARE_BUF_SZ); |
455
|
0
|
|
|
|
|
|
buf = buf0; |
456
|
|
|
|
|
|
|
} else { |
457
|
0
|
|
|
|
|
|
nread0 = COMPARE_BUF_SZ; |
458
|
0
|
0
|
|
|
|
|
if ( nread0 > info->bufferIdx - info->matchPosn ) nread0 = info->bufferIdx - info->matchPosn; |
459
|
0
|
|
|
|
|
|
buf = info->buffer + info->matchPosn; |
460
|
|
|
|
|
|
|
} |
461
|
0
|
0
|
|
|
|
|
for ( i = 0 ; i < BPC_POOL_WRITE_CONCURRENT_MATCH ; i++ ) { |
462
|
|
|
|
|
|
|
OFF_T nread1; |
463
|
|
|
|
|
|
|
|
464
|
0
|
0
|
|
|
|
|
if ( !info->match[i].used ) continue; |
465
|
0
|
|
|
|
|
|
nMatch++; |
466
|
|
|
|
|
|
|
/* |
467
|
|
|
|
|
|
|
* Try to read an extra byte when we expect EOF, to make sure the candidate file is also at EOF |
468
|
|
|
|
|
|
|
*/ |
469
|
0
|
|
|
|
|
|
nread1 = bpc_fileZIO_read(&info->match[i].fd, buf1, nread0 > 0 ? nread0 : 1); |
470
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 9 ) bpc_logMsgf("Read %d bytes of %d from match[%d] (%s)\n", (int)nread1, (int)nread0, i, info->match[i].fileName); |
471
|
0
|
0
|
|
|
|
|
if ( nread0 != nread1 || (nread0 > 0 && memcmp(buf, buf1, nread0)) ) { |
|
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
472
|
0
|
|
|
|
|
|
bpc_fileZIO_close(&info->match[i].fd); |
473
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 8 ) bpc_logMsgf("match[%d] no longer matches\n", i); |
474
|
0
|
|
|
|
|
|
info->match[i].used = 0; |
475
|
0
|
|
|
|
|
|
replaceCnt++; |
476
|
|
|
|
|
|
|
} |
477
|
|
|
|
|
|
|
} |
478
|
0
|
|
|
|
|
|
info->matchPosn += nread0; |
479
|
|
|
|
|
|
|
|
480
|
0
|
0
|
|
|
|
|
if ( replaceCnt ) { |
481
|
0
|
|
|
|
|
|
nMatch = bpc_poolWrite_updateMatches(info); |
482
|
|
|
|
|
|
|
} |
483
|
0
|
0
|
|
|
|
|
if ( nread0 == 0 || nMatch == 0 ) { |
|
|
0
|
|
|
|
|
|
484
|
|
|
|
|
|
|
/* |
485
|
|
|
|
|
|
|
* we are at eof (with a match) or there are no matches |
486
|
|
|
|
|
|
|
*/ |
487
|
0
|
|
|
|
|
|
info->state = 4; |
488
|
0
|
|
|
|
|
|
break; |
489
|
|
|
|
|
|
|
} |
490
|
0
|
|
|
|
|
|
} |
491
|
|
|
|
|
|
|
} |
492
|
0
|
0
|
|
|
|
|
if ( info->state == 4 ) { |
493
|
|
|
|
|
|
|
/* |
494
|
|
|
|
|
|
|
* see if there is a matching file |
495
|
|
|
|
|
|
|
*/ |
496
|
0
|
|
|
|
|
|
int i, nMatch = 0, iMatch = 0; |
497
|
|
|
|
|
|
|
|
498
|
0
|
0
|
|
|
|
|
for ( i = BPC_POOL_WRITE_CONCURRENT_MATCH -1 ; i >= 0 ; i-- ) { |
499
|
0
|
0
|
|
|
|
|
if ( !info->match[i].used ) continue; |
500
|
0
|
|
|
|
|
|
nMatch++; |
501
|
0
|
|
|
|
|
|
iMatch = i; |
502
|
|
|
|
|
|
|
} |
503
|
0
|
0
|
|
|
|
|
if ( nMatch == 0 ) { |
504
|
|
|
|
|
|
|
ssize_t writeRet; |
505
|
|
|
|
|
|
|
/* |
506
|
|
|
|
|
|
|
* Need to write a new file if not written already |
507
|
|
|
|
|
|
|
*/ |
508
|
0
|
0
|
|
|
|
|
if ( !info->fdOpen && info->fileSize > 0 ) { |
|
|
0
|
|
|
|
|
|
509
|
0
|
0
|
|
|
|
|
if ( bpc_fileZIO_open(&info->fd, info->tmpFileName, 1, info->compress) ) { |
510
|
0
|
|
|
|
|
|
info->errorCnt++; |
511
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_write: can't open/create %s for writing", info->tmpFileName); |
512
|
0
|
|
|
|
|
|
return -1; |
513
|
|
|
|
|
|
|
} |
514
|
0
|
0
|
|
|
|
|
if ( info->bufferIdx > 0 ) { |
515
|
0
|
0
|
|
|
|
|
if ( (writeRet = bpc_fileZIO_write(&info->fd, info->buffer, info->bufferIdx)) != (ssize_t)info->bufferIdx ) { |
516
|
0
|
|
|
|
|
|
info->errorCnt++; |
517
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_write: write of %u bytes to %s failed, return = %d", |
518
|
0
|
|
|
|
|
|
info->bufferIdx, info->tmpFileName, (int)writeRet); |
519
|
0
|
|
|
|
|
|
return -1; |
520
|
|
|
|
|
|
|
} |
521
|
|
|
|
|
|
|
} |
522
|
0
|
|
|
|
|
|
bpc_fileZIO_close(&info->fd); |
523
|
|
|
|
|
|
|
} |
524
|
0
|
0
|
|
|
|
|
if ( info->fileSize > 0 ) { |
525
|
|
|
|
|
|
|
char hexStr[BPC_DIGEST_LEN_MAX * 2 + 1]; |
526
|
0
|
|
|
|
|
|
bpc_digest_append_ext(&info->digest, 0); |
527
|
0
|
|
|
|
|
|
bpc_digest_digest2str(&info->digest, hexStr); |
528
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 5 ) bpc_logMsgf("No match... adding %s to pool (digest = %s)\n", info->tmpFileName, hexStr); |
529
|
0
|
|
|
|
|
|
bpc_poolWrite_addToPool(info, info->tmpFileName, 0); |
530
|
|
|
|
|
|
|
} else { |
531
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 5 ) bpc_logMsgf("Zero length file - don't match anything\n"); |
532
|
0
|
|
|
|
|
|
info->digest.len = 0; |
533
|
0
|
|
|
|
|
|
info->retValue = 1; |
534
|
0
|
|
|
|
|
|
info->poolFileSize = 0; |
535
|
|
|
|
|
|
|
} |
536
|
|
|
|
|
|
|
} else { |
537
|
|
|
|
|
|
|
/* |
538
|
|
|
|
|
|
|
* We matched a pool file |
539
|
|
|
|
|
|
|
*/ |
540
|
0
|
0
|
|
|
|
|
if ( nMatch > 1 ) { |
541
|
|
|
|
|
|
|
char hexStr[BPC_DIGEST_LEN_MAX * 2 + 1]; |
542
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 4 ) bpc_logMsgf("Botch - got multiple pool file matches\n"); |
543
|
0
|
|
|
|
|
|
info->errorCnt++; |
544
|
0
|
|
|
|
|
|
bpc_digest_digest2str(&info->digest, hexStr); |
545
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_write: got %d matching files for digest %s\n", nMatch, hexStr); |
546
|
|
|
|
|
|
|
} |
547
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 7 ) bpc_logMsgf("Found match with match[%d] (%s)\n", iMatch, info->match[iMatch].fileName); |
548
|
0
|
0
|
|
|
|
|
if ( info->match[iMatch].v3File ) { |
549
|
0
|
|
|
|
|
|
bpc_digest_append_ext(&info->digest, 0); |
550
|
0
|
|
|
|
|
|
bpc_poolWrite_addToPool(info, info->match[iMatch].fileName, info->match[iMatch].v3File); |
551
|
|
|
|
|
|
|
} else { |
552
|
0
|
|
|
|
|
|
info->digest = info->match[iMatch].digest; |
553
|
0
|
|
|
|
|
|
info->retValue = 1; |
554
|
0
|
|
|
|
|
|
info->poolFileSize = info->match[iMatch].fileSize; |
555
|
|
|
|
|
|
|
} |
556
|
0
|
0
|
|
|
|
|
if ( info->fdOpen ) { |
557
|
0
|
|
|
|
|
|
bpc_fileZIO_close(&info->fd); |
558
|
0
|
|
|
|
|
|
unlink(info->tmpFileName); |
559
|
0
|
|
|
|
|
|
info->fdOpen = 0; |
560
|
|
|
|
|
|
|
} |
561
|
|
|
|
|
|
|
} |
562
|
|
|
|
|
|
|
} |
563
|
0
|
|
|
|
|
|
return 0; |
564
|
|
|
|
|
|
|
} |
565
|
|
|
|
|
|
|
|
566
|
0
|
|
|
|
|
|
int bpc_poolWrite_createPoolDir(bpc_poolWrite_info *info, bpc_digest *digest) |
567
|
|
|
|
|
|
|
{ |
568
|
|
|
|
|
|
|
char path[BPC_MAXPATHLEN], *p; |
569
|
|
|
|
|
|
|
int ret; |
570
|
|
|
|
|
|
|
|
571
|
|
|
|
|
|
|
/* |
572
|
|
|
|
|
|
|
* get the full path, and prune off the file name to get the directory |
573
|
|
|
|
|
|
|
*/ |
574
|
0
|
|
|
|
|
|
bpc_digest_md52path(path, info->compress, digest); |
575
|
0
|
0
|
|
|
|
|
if ( !(p = strrchr(path, '/')) ) { |
576
|
0
|
|
|
|
|
|
info->errorCnt++; |
577
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_createPoolDir: can't find trailing / in path %s", path); |
578
|
0
|
|
|
|
|
|
return -1; |
579
|
|
|
|
|
|
|
} |
580
|
0
|
|
|
|
|
|
*p = '\0'; |
581
|
|
|
|
|
|
|
|
582
|
0
|
0
|
|
|
|
|
if ( (ret = bpc_path_create(path)) ) { |
583
|
0
|
|
|
|
|
|
info->errorCnt++; |
584
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_createPoolDir: can't create directory path %s", path); |
585
|
|
|
|
|
|
|
} |
586
|
0
|
|
|
|
|
|
return ret; |
587
|
|
|
|
|
|
|
} |
588
|
|
|
|
|
|
|
|
589
|
0
|
|
|
|
|
|
void bpc_poolWrite_cleanup(bpc_poolWrite_info *info) |
590
|
|
|
|
|
|
|
{ |
591
|
|
|
|
|
|
|
int i; |
592
|
|
|
|
|
|
|
|
593
|
0
|
0
|
|
|
|
|
if ( info->fdOpen ) bpc_fileZIO_close(&info->fd); |
594
|
0
|
|
|
|
|
|
info->fdOpen = 0; |
595
|
|
|
|
|
|
|
|
596
|
0
|
0
|
|
|
|
|
while ( info->candidateList ) { |
597
|
0
|
|
|
|
|
|
bpc_candidate_file *candidateFile = info->candidateList; |
598
|
0
|
|
|
|
|
|
info->candidateList = candidateFile->next; |
599
|
0
|
|
|
|
|
|
free(candidateFile); |
600
|
|
|
|
|
|
|
} |
601
|
0
|
0
|
|
|
|
|
for ( i = 0 ; i < BPC_POOL_WRITE_CONCURRENT_MATCH ; i++ ) { |
602
|
0
|
0
|
|
|
|
|
if ( !info->match[i].used ) continue; |
603
|
0
|
|
|
|
|
|
bpc_fileZIO_close(&info->match[i].fd); |
604
|
0
|
|
|
|
|
|
info->match[i].used = 0; |
605
|
|
|
|
|
|
|
} |
606
|
0
|
0
|
|
|
|
|
if ( info->buffer ) { |
607
|
0
|
|
|
|
|
|
*(void**)info->buffer = DataBufferFreeList; |
608
|
0
|
|
|
|
|
|
DataBufferFreeList = info->buffer; |
609
|
0
|
|
|
|
|
|
info->buffer = NULL; |
610
|
|
|
|
|
|
|
} |
611
|
0
|
|
|
|
|
|
} |
612
|
|
|
|
|
|
|
|
613
|
|
|
|
|
|
|
/* |
614
|
|
|
|
|
|
|
* Called after the data is written. The return information is passed via four arguments: |
615
|
|
|
|
|
|
|
* |
616
|
|
|
|
|
|
|
* (match, digest, poolFileSize, errorCnt) |
617
|
|
|
|
|
|
|
* |
618
|
|
|
|
|
|
|
* The return values are: |
619
|
|
|
|
|
|
|
* |
620
|
|
|
|
|
|
|
* - match: |
621
|
|
|
|
|
|
|
* If match == 0, then the file doesn't match either the new or old pools. |
622
|
|
|
|
|
|
|
* The file has been added to the pool. |
623
|
|
|
|
|
|
|
* |
624
|
|
|
|
|
|
|
* If match == 1, then the file matches the new pool. No file is |
625
|
|
|
|
|
|
|
* written. |
626
|
|
|
|
|
|
|
* |
627
|
|
|
|
|
|
|
* If match == 2, then the file matches the old pool. The old pool |
628
|
|
|
|
|
|
|
* file was moved to become the new pool file. |
629
|
|
|
|
|
|
|
* |
630
|
|
|
|
|
|
|
* - digest: the 16+ byte binary MD5 digest, possibly appended with |
631
|
|
|
|
|
|
|
* on or more additional bytes to point to the right pool file in |
632
|
|
|
|
|
|
|
* case there are MD5 collisions |
633
|
|
|
|
|
|
|
* |
634
|
|
|
|
|
|
|
* - poolFileSize: the compressed pool file size |
635
|
|
|
|
|
|
|
* |
636
|
|
|
|
|
|
|
* - errorCnt: number of errors |
637
|
|
|
|
|
|
|
*/ |
638
|
0
|
|
|
|
|
|
void bpc_poolWrite_close(bpc_poolWrite_info *info, int *match, bpc_digest *digest, OFF_T *poolFileSize, int *errorCnt) |
639
|
|
|
|
|
|
|
{ |
640
|
0
|
|
|
|
|
|
bpc_poolWrite_write(info, NULL, 0); |
641
|
0
|
|
|
|
|
|
bpc_poolWrite_cleanup(info); |
642
|
0
|
|
|
|
|
|
*match = info->retValue; |
643
|
0
|
|
|
|
|
|
*digest = info->digest; |
644
|
0
|
|
|
|
|
|
*poolFileSize = info->poolFileSize; |
645
|
0
|
|
|
|
|
|
*errorCnt = info->errorCnt; |
646
|
0
|
|
|
|
|
|
} |
647
|
|
|
|
|
|
|
|
648
|
0
|
|
|
|
|
|
void bpc_poolWrite_repeatPoolWrite(bpc_poolWrite_info *info, char *fileNameTmp) |
649
|
|
|
|
|
|
|
{ |
650
|
0
|
|
|
|
|
|
bpc_poolWrite_cleanup(info); |
651
|
|
|
|
|
|
|
|
652
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 5 ) bpc_logMsgf("bpc_poolWrite_repeatPoolWrite: rewriting %s\n", fileNameTmp); |
653
|
0
|
0
|
|
|
|
|
if ( info->retryCnt++ > 8 ) { |
654
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_repeatPoolWrite: giving up on %s after %d attempts\n", fileNameTmp, info->retryCnt); |
655
|
0
|
|
|
|
|
|
info->errorCnt++; |
656
|
0
|
|
|
|
|
|
unlink(fileNameTmp); |
657
|
0
|
|
|
|
|
|
return; |
658
|
|
|
|
|
|
|
} |
659
|
0
|
|
|
|
|
|
strcpy(info->tmpFileName, fileNameTmp); |
660
|
0
|
0
|
|
|
|
|
if ( bpc_fileZIO_open(&info->fd, fileNameTmp, 0, info->compress) < 0 ) { |
661
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_repeatPoolWrite: can't open %s for reading", fileNameTmp); |
662
|
0
|
|
|
|
|
|
info->errorCnt++; |
663
|
0
|
|
|
|
|
|
return; |
664
|
|
|
|
|
|
|
} |
665
|
0
|
|
|
|
|
|
info->eof = 1; |
666
|
0
|
|
|
|
|
|
info->state = 2; |
667
|
0
|
|
|
|
|
|
info->fdOpen = 1; |
668
|
0
|
|
|
|
|
|
bpc_poolWrite_write(info, NULL, 0); |
669
|
|
|
|
|
|
|
} |
670
|
|
|
|
|
|
|
|
671
|
0
|
|
|
|
|
|
int bpc_poolWrite_copyToPool(bpc_poolWrite_info *info, char *poolPath, char *fileName) |
672
|
|
|
|
|
|
|
{ |
673
|
|
|
|
|
|
|
int fdRead, fdWrite; |
674
|
|
|
|
|
|
|
int nRead, nWrite; |
675
|
|
|
|
|
|
|
|
676
|
0
|
0
|
|
|
|
|
if ( (fdWrite = open(poolPath, O_WRONLY | O_CREAT | O_EXCL, 0666)) < 0 ) { |
677
|
0
|
|
|
|
|
|
info->errorCnt++; |
678
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_copyToPool: can't open/create %s for writing", poolPath); |
679
|
0
|
|
|
|
|
|
return -1; |
680
|
|
|
|
|
|
|
} |
681
|
0
|
0
|
|
|
|
|
if ( (fdRead = open(fileName, O_RDONLY)) < 0 ) { |
682
|
0
|
|
|
|
|
|
info->errorCnt++; |
683
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_copyToPool: can't open %s for reading", fileName); |
684
|
0
|
|
|
|
|
|
return -1; |
685
|
|
|
|
|
|
|
} |
686
|
|
|
|
|
|
|
|
687
|
0
|
0
|
|
|
|
|
while ( (nRead = read(fdRead, info->buffer, sizeof(info->buffer))) > 0 ) { |
688
|
0
|
|
|
|
|
|
uchar *p = info->buffer; |
689
|
|
|
|
|
|
|
int thisWrite; |
690
|
|
|
|
|
|
|
|
691
|
0
|
|
|
|
|
|
nWrite = 0; |
692
|
0
|
0
|
|
|
|
|
while ( nWrite < nRead ) { |
693
|
|
|
|
|
|
|
do { |
694
|
0
|
|
|
|
|
|
thisWrite = write(fdWrite, p, nRead - nWrite); |
695
|
0
|
0
|
|
|
|
|
} while ( thisWrite < 0 && errno == EINTR ); |
|
|
0
|
|
|
|
|
|
696
|
0
|
0
|
|
|
|
|
if ( thisWrite < 0 ) { |
697
|
0
|
|
|
|
|
|
info->errorCnt++; |
698
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_copyToPool: write to %s failed (errno = %d)", poolPath, errno); |
699
|
0
|
|
|
|
|
|
close(fdWrite); |
700
|
0
|
|
|
|
|
|
close(fdRead); |
701
|
0
|
|
|
|
|
|
unlink(poolPath); |
702
|
0
|
|
|
|
|
|
return -1; |
703
|
|
|
|
|
|
|
} |
704
|
0
|
|
|
|
|
|
p += thisWrite; |
705
|
0
|
|
|
|
|
|
nWrite += thisWrite; |
706
|
|
|
|
|
|
|
} |
707
|
|
|
|
|
|
|
} |
708
|
0
|
|
|
|
|
|
close(fdWrite); |
709
|
0
|
|
|
|
|
|
close(fdRead); |
710
|
0
|
|
|
|
|
|
return 0; |
711
|
|
|
|
|
|
|
} |
712
|
|
|
|
|
|
|
|
713
|
0
|
|
|
|
|
|
void bpc_poolWrite_addToPool(bpc_poolWrite_info *info, char *fileName, int v3PoolFile) |
714
|
|
|
|
|
|
|
{ |
715
|
|
|
|
|
|
|
STRUCT_STAT st; |
716
|
|
|
|
|
|
|
char poolPath[BPC_MAXPATHLEN]; |
717
|
0
|
|
|
|
|
|
int redo = 0; |
718
|
|
|
|
|
|
|
|
719
|
0
|
0
|
|
|
|
|
if ( bpc_poolWrite_createPoolDir(info, &info->digest) ) return; |
720
|
|
|
|
|
|
|
|
721
|
|
|
|
|
|
|
/* |
722
|
|
|
|
|
|
|
* If originally present, make sure the zero-length file is still there (and still |
723
|
|
|
|
|
|
|
* zero-length), and the open slot is still open. If not, it probably means someone |
724
|
|
|
|
|
|
|
* beat us to it, and we should re-do the whole pool matching to see if the newly |
725
|
|
|
|
|
|
|
* added pool file now matches. |
726
|
|
|
|
|
|
|
*/ |
727
|
0
|
0
|
|
|
|
|
if ( info->digestExtZeroLen >= 0 ) { |
728
|
0
|
|
|
|
|
|
bpc_digest_append_ext(&info->digest, info->digestExtZeroLen); |
729
|
0
|
|
|
|
|
|
bpc_digest_md52path(poolPath, info->compress, &info->digest); |
730
|
0
|
0
|
|
|
|
|
if ( stat(poolPath, &st) || st.st_size != 0 ) { |
|
|
0
|
|
|
|
|
|
731
|
0
|
|
|
|
|
|
redo = 1; |
732
|
|
|
|
|
|
|
} |
733
|
|
|
|
|
|
|
} |
734
|
0
|
0
|
|
|
|
|
if ( !redo ) { |
735
|
0
|
|
|
|
|
|
bpc_digest_append_ext(&info->digest, info->digestExtOpen); |
736
|
0
|
|
|
|
|
|
bpc_digest_md52path(poolPath, info->compress, &info->digest); |
737
|
0
|
0
|
|
|
|
|
if ( !stat(poolPath, &st) ) { |
738
|
0
|
|
|
|
|
|
redo = 1; |
739
|
|
|
|
|
|
|
} |
740
|
|
|
|
|
|
|
} |
741
|
|
|
|
|
|
|
|
742
|
|
|
|
|
|
|
/* |
743
|
|
|
|
|
|
|
* Try to insert the new file at the zero-length file slot (if present). |
744
|
|
|
|
|
|
|
*/ |
745
|
0
|
0
|
|
|
|
|
if ( !redo && info->digestExtZeroLen >= 0 ) { |
|
|
0
|
|
|
|
|
|
746
|
|
|
|
|
|
|
char lockFile[BPC_MAXPATHLEN]; |
747
|
|
|
|
|
|
|
int lockFd; |
748
|
|
|
|
|
|
|
/* |
749
|
|
|
|
|
|
|
* We can replace a zero-length file, but only via locking to |
750
|
|
|
|
|
|
|
* avoid race conditions. Since the hardlinking code below doesn't |
751
|
|
|
|
|
|
|
* use a lock, we can't remove the file and use a hardlink |
752
|
|
|
|
|
|
|
* because of race conditions - another process might be |
753
|
|
|
|
|
|
|
* inserting with the same digest and grab the slot. |
754
|
|
|
|
|
|
|
* |
755
|
|
|
|
|
|
|
* So we make sure we have exclusive access via a lock file, |
756
|
|
|
|
|
|
|
* check that the file is still zero-length, and then rename |
757
|
|
|
|
|
|
|
* the file. If that fails then we redo everything. |
758
|
|
|
|
|
|
|
*/ |
759
|
0
|
|
|
|
|
|
bpc_digest_append_ext(&info->digest, info->digestExtZeroLen); |
760
|
0
|
|
|
|
|
|
bpc_digest_md52path(poolPath, info->compress, &info->digest); |
761
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_poolWrite_addToPool: replacing empty pool file %s with %s\n", poolPath, fileName); |
762
|
0
|
|
|
|
|
|
snprintf(lockFile, BPC_MAXPATHLEN, "%s.lock", poolPath); |
763
|
0
|
|
|
|
|
|
lockFd = bpc_lockRangeFile(lockFile, 0, 1, 1); |
764
|
|
|
|
|
|
|
/* |
765
|
|
|
|
|
|
|
* If we don't have the lock, or the file is no longer zero length, or the rename fails, |
766
|
|
|
|
|
|
|
* then try again. |
767
|
|
|
|
|
|
|
*/ |
768
|
0
|
0
|
|
|
|
|
if ( lockFd < 0 || stat(poolPath, &st) || st.st_size != 0 || rename(fileName, poolPath) ) { |
|
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
769
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 5 ) { |
770
|
0
|
|
|
|
|
|
bpc_logMsgf("bpc_poolWrite_addToPool: lock/rename failed: need to repeat write (lockFd = %d, size = %lu, errno = %d)\n", |
771
|
0
|
|
|
|
|
|
lockFd, (unsigned long)st.st_size, errno); |
772
|
|
|
|
|
|
|
} |
773
|
0
|
0
|
|
|
|
|
if ( lockFd >= 0 ) { |
774
|
0
|
|
|
|
|
|
bpc_unlockRangeFile(lockFd); |
775
|
|
|
|
|
|
|
} |
776
|
0
|
|
|
|
|
|
unlink(lockFile); |
777
|
0
|
|
|
|
|
|
redo = 1; |
778
|
|
|
|
|
|
|
} else { |
779
|
0
|
|
|
|
|
|
chmod(poolPath, 0444); |
780
|
0
|
|
|
|
|
|
stat(poolPath, &st); |
781
|
0
|
0
|
|
|
|
|
info->retValue = v3PoolFile ? 2 : 0; |
782
|
0
|
|
|
|
|
|
info->poolFileSize = st.st_size; |
783
|
0
|
|
|
|
|
|
bpc_unlockRangeFile(lockFd); |
784
|
0
|
|
|
|
|
|
unlink(lockFile); |
785
|
0
|
|
|
|
|
|
return; |
786
|
|
|
|
|
|
|
} |
787
|
|
|
|
|
|
|
} |
788
|
|
|
|
|
|
|
|
789
|
|
|
|
|
|
|
/* |
790
|
|
|
|
|
|
|
* Now try to link the file to the new empty slot at the end |
791
|
|
|
|
|
|
|
*/ |
792
|
0
|
0
|
|
|
|
|
if ( !redo ) { |
793
|
|
|
|
|
|
|
int linkOk, statOk; |
794
|
|
|
|
|
|
|
ino_t fileIno, poolIno; |
795
|
|
|
|
|
|
|
/* |
796
|
|
|
|
|
|
|
* Since this is a new slot, there is no need to do locking since |
797
|
|
|
|
|
|
|
* the link or open operations below are atomic/exclusive. |
798
|
|
|
|
|
|
|
* |
799
|
|
|
|
|
|
|
* First try to hardlink to the empty pool file slot |
800
|
|
|
|
|
|
|
*/ |
801
|
0
|
|
|
|
|
|
bpc_digest_append_ext(&info->digest, info->digestExtOpen); |
802
|
0
|
|
|
|
|
|
bpc_digest_md52path(poolPath, info->compress, &info->digest); |
803
|
0
|
0
|
|
|
|
|
if ( stat(fileName, &st) ) { |
804
|
0
|
|
|
|
|
|
info->errorCnt++; |
805
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_addToPool: can't stat %s\n", fileName); |
806
|
0
|
|
|
|
|
|
return; |
807
|
|
|
|
|
|
|
} |
808
|
0
|
|
|
|
|
|
fileIno = st.st_ino; |
809
|
0
|
|
|
|
|
|
linkOk = !link(fileName, poolPath); |
810
|
0
|
0
|
|
|
|
|
if ( !(statOk = !stat(poolPath, &st)) ) linkOk = 0; |
811
|
0
|
|
|
|
|
|
poolIno = st.st_ino; |
812
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_poolWrite_addToPool: link %s -> %s (linkOk = %d, statOk = %d, ino = %lu/%lu)\n", |
813
|
|
|
|
|
|
|
poolPath, fileName, linkOk, statOk, (unsigned long)fileIno, (unsigned long)poolIno); |
814
|
|
|
|
|
|
|
|
815
|
|
|
|
|
|
|
/* |
816
|
|
|
|
|
|
|
* make sure the link really worked by checking inode numbers |
817
|
|
|
|
|
|
|
* TODO: test these different cases. |
818
|
|
|
|
|
|
|
*/ |
819
|
0
|
0
|
|
|
|
|
if ( statOk && fileIno == poolIno ) { |
|
|
0
|
|
|
|
|
|
820
|
|
|
|
|
|
|
/* |
821
|
|
|
|
|
|
|
* remove the original file and return |
822
|
|
|
|
|
|
|
*/ |
823
|
0
|
|
|
|
|
|
unlink(fileName); |
824
|
0
|
|
|
|
|
|
chmod(poolPath, 0444); |
825
|
0
|
0
|
|
|
|
|
info->retValue = v3PoolFile ? 2 : 0; |
826
|
0
|
|
|
|
|
|
info->poolFileSize = st.st_size; |
827
|
0
|
|
|
|
|
|
return; |
828
|
|
|
|
|
|
|
} |
829
|
|
|
|
|
|
|
/* |
830
|
|
|
|
|
|
|
* Something failed. If the stat failed, the hardlink failure wasn't due |
831
|
|
|
|
|
|
|
* to another file being added by someone else. Perhaps the cpool is |
832
|
|
|
|
|
|
|
* split across multiple file systems? |
833
|
|
|
|
|
|
|
*/ |
834
|
0
|
0
|
|
|
|
|
if ( !statOk ) { |
835
|
|
|
|
|
|
|
/* |
836
|
|
|
|
|
|
|
* The hardlink failed. This could be due to hitting the hardlink |
837
|
|
|
|
|
|
|
* limit, or the fact that fileName and poolPath are on different |
838
|
|
|
|
|
|
|
* file systems, or the fileName didn't get written. |
839
|
|
|
|
|
|
|
* Just copy the file instead (assuming fileName got written). |
840
|
|
|
|
|
|
|
*/ |
841
|
0
|
|
|
|
|
|
bpc_poolWrite_copyToPool(info, poolPath, fileName); |
842
|
0
|
|
|
|
|
|
return; |
843
|
|
|
|
|
|
|
} |
844
|
|
|
|
|
|
|
} |
845
|
|
|
|
|
|
|
|
846
|
|
|
|
|
|
|
/* |
847
|
|
|
|
|
|
|
* We need to redo the pool write, since it appears someone else has added |
848
|
|
|
|
|
|
|
* a pool file with the same digest. |
849
|
|
|
|
|
|
|
*/ |
850
|
0
|
|
|
|
|
|
bpc_poolWrite_repeatPoolWrite(info, fileName); |
851
|
|
|
|
|
|
|
} |
852
|
|
|
|
|
|
|
|
853
|
|
|
|
|
|
|
/* |
854
|
|
|
|
|
|
|
* Safely remove the o+x permission that marks a file for future deletion. |
855
|
|
|
|
|
|
|
* Similar locking is done by BackupPC_refCountUpdate so we can avoid any |
856
|
|
|
|
|
|
|
* race conditions with the file actually being deleted. |
857
|
|
|
|
|
|
|
* |
858
|
|
|
|
|
|
|
* Returns 0 on success. |
859
|
|
|
|
|
|
|
*/ |
860
|
0
|
|
|
|
|
|
int bpc_poolWrite_unmarkPendingDelete(char *poolPath) |
861
|
|
|
|
|
|
|
{ |
862
|
|
|
|
|
|
|
char lockFile[BPC_MAXPATHLEN], *p; |
863
|
|
|
|
|
|
|
STRUCT_STAT st; |
864
|
|
|
|
|
|
|
int lockFd; |
865
|
|
|
|
|
|
|
|
866
|
|
|
|
|
|
|
/* |
867
|
|
|
|
|
|
|
* The lock file is in the first level of pool sub directories - one level |
868
|
|
|
|
|
|
|
* up from the full path. So we need to find the 2nd last '/'. |
869
|
|
|
|
|
|
|
*/ |
870
|
0
|
|
|
|
|
|
snprintf(lockFile, BPC_MAXPATHLEN, "%s", poolPath); |
871
|
0
|
0
|
|
|
|
|
if ( !(p = strrchr(lockFile, '/')) ) return -1; |
872
|
0
|
|
|
|
|
|
*p = '\0'; |
873
|
0
|
0
|
|
|
|
|
if ( !(p = strrchr(lockFile, '/')) ) return -1; |
874
|
0
|
|
|
|
|
|
snprintf(p + 1, BPC_MAXPATHLEN - (p + 1 - lockFile), "%s", "LOCK"); |
875
|
0
|
0
|
|
|
|
|
if ( (lockFd = bpc_lockRangeFile(lockFile, 0, 1, 1)) < 0 ) return -1; |
876
|
0
|
0
|
|
|
|
|
if ( !stat(poolPath, &st) && !chmod(poolPath, st.st_mode & ~S_IXOTH & ~S_IFMT) ) { |
|
|
0
|
|
|
|
|
|
877
|
0
|
0
|
|
|
|
|
if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_poolWrite_unmarkPendingDelete(%s) succeeded\n", poolPath); |
878
|
0
|
|
|
|
|
|
bpc_unlockRangeFile(lockFd); |
879
|
0
|
|
|
|
|
|
return 0; |
880
|
|
|
|
|
|
|
} else { |
881
|
0
|
|
|
|
|
|
bpc_logErrf("bpc_poolWrite_unmarkPendingDelete(%s) failed; errno = %d\n", poolPath, errno); |
882
|
0
|
|
|
|
|
|
bpc_unlockRangeFile(lockFd); |
883
|
0
|
|
|
|
|
|
return -1; |
884
|
|
|
|
|
|
|
} |
885
|
|
|
|
|
|
|
} |