Context Navigation

← Previous Revision
Next Revision →
Blame
Revision Log

arb_zfile.cxx

Visit:

Last change on this file was 18821, checked in by westram, 3 years ago

ignore SIGPIPE while ArbDBWriter exists

arb does no longer terminate, but instead prints error, when pipe breaks, e.g.:

19:01.38  While writing '/home/user/data/test with space.arb~': Broken pipe
          (close reports: pipe writing to
           file='/home/user/data/test with space.arb~'
           using cmd='xz > /home/user/data/test with space.arb~'
           failed with exitcode=1 (broken pipe? corrupted archive?)
           (res=256, exited=1, signaled=0, status=1))

File size: 11.8 KB

Line
1	// ================================================================ //
2	// //
3	// File : arb_zfile.cxx //
4	// Purpose : Compressed file I/O //
5	// //
6	// Coded by Ralf Westram (coder@reallysoft.de) in November 2015 //
7	// http://www.arb-home.de/ //
8	// //
9	// ================================================================ //
10
11	#include "arb_zfile.h"
12	#include "arb_file.h"
13	#include "arb_msg.h"
14	#include "arb_misc.h"
15	#include "arb_string.h"
16
17	#include <string>
18	#include <map>
19
20	using namespace std;
21
22	class zinfo {
23	// info stored for each sucessfully opened file
24	// to support proper error message on close.
25	bool writing; // false -> reading
26	string filename;
27	string pipe_cmd;
28	public:
29	zinfo() {}
30	zinfo(bool writing_, const char filename_, const char pipe_cmd_)
31	: writing(writing_),
32	filename(filename_),
33	pipe_cmd(pipe_cmd_)
34	{}
35
36	bool isOutputPipe() const { return writing; }
37	const char *get_filename() const { return filename.c_str(); }
38	const char *get_pipecmd() const { return pipe_cmd.c_str(); }
39	};
40	static map<FILE*,zinfo> zfile_info;
41
42	FILE ARB_zfopen(const char name, const char *mode, FileCompressionMode cmode, GB_ERROR& error, bool hideStderr) {
43	arb_assert(!error);
44
45	if (strchr(mode, 'a')) {
46	error = "Cannot append to file using ARB_zfopen";
47	return NULp;
48	}
49	if (strchr(mode, 't')) {
50	error = "Cannot use textmode for ARB_zfopen";
51	return NULp;
52	}
53	if (strchr(mode, '+')) {
54	error = "Cannot open file in read and write mode with ARB_zfopen";
55	return NULp;
56	}
57
58	bool forOutput = strchr(mode, 'w');
59	FILE *fp = NULp;
60
61	if (cmode == ZFILE_AUTODETECT) {
62	if (forOutput) {
63	error = "Autodetecting compression mode only works for input files";
64	}
65	else {
66	fp = fopen(name, "rb");
67	if (!fp) error = GB_IO_error("opening", name);
68	else {
69	// detect compression and set 'cmode'
70	const size_t MAGICSIZE = 5;
71	char buffer[MAGICSIZE];
72
73	size_t bytes_read = fread(buffer, 1, MAGICSIZE, fp);
74	fclose(fp);
75	fp = NULp;
76
77	if (bytes_read>=2 && strncmp(buffer, "\x1f\x8b", 2) == 0) cmode = ZFILE_GZIP;
78	else if (bytes_read>=2 && strncmp(buffer, "BZ", 2) == 0) cmode = ZFILE_BZIP2;
79	else if (bytes_read>=5 && strncmp(buffer, "\xfd" "7zXZ", 5) == 0) cmode = ZFILE_XZ;
80	else {
81	cmode = ZFILE_UNCOMPRESSED;
82	}
83	}
84	}
85	}
86
87	if (cmode == ZFILE_UNCOMPRESSED) {
88	fp = fopen(name, mode);
89	if (!fp) error = GB_IO_error("opening", name);
90	else {
91	zfile_info[fp] = zinfo(forOutput, name, "");
92	}
93	}
94	else {
95	if (!error) {
96	const char *compressor = NULp; // command used to compress (and decompress)
97	const char *decompress_flag = "-d"; // flag needed to decompress (assumes none to compress)
98
99	switch (cmode) {
100	case ZFILE_GZIP: {
101	static char *pigz = ARB_executable("pigz", ARB_getenv_ignore_empty("PATH"));
102	if (pigz) {
103	ARB_warn_about_unwanted_chars(pigz, "the pigz executable (detected as gzip dropin)");
104	compressor = pigz;
105	}
106	else {
107	compressor = "gzip";
108	}
109	break;
110	}
111	case ZFILE_BZIP2: compressor = "bzip2"; break;
112	case ZFILE_XZ: compressor = "xz"; break;
113
114	default:
115	error = GBS_global_string("Invalid compression mode (%i)", int(cmode));
116	break;
117
118	#if defined(USE_BROKEN_COMPRESSION)
119	case ZFILE_BROKEN:
120	compressor = "arb_weirdo"; // a non-existing command!
121	break;
122	#endif
123	}
124
125	if (!error) {
126	char *pipeCmd = forOutput
127	? GBS_global_string_copy("%s > %s", compressor, name)
128	: GBS_global_string_copy("%s %s < %s", compressor, decompress_flag, name);
129
130	if (hideStderr) {
131	freeset(pipeCmd, GBS_global_string_copy("( %s 2>/dev/null )", pipeCmd));
132	}
133
134	// remove 'b' from mode (pipes are binary by default)
135	char *impl_b_mode = ARB_strdup(mode);
136	while (1) {
137	char *b = strchr(impl_b_mode, 'b');
138	if (!b) break;
139	strcpy(b, b+1);
140	}
141
142	if (forOutput) { // write to pipe
143	fp = popen(pipeCmd, impl_b_mode);
144	if (!fp) error = GB_IO_error("writing to pipe", pipeCmd);
145	}
146	else { // read from pipe
147	fp = popen(pipeCmd, impl_b_mode);
148	if (!fp) error = GB_IO_error("reading from pipe", pipeCmd);
149	}
150
151	if (!error) {
152	zfile_info[fp] = zinfo(forOutput, name, pipeCmd);
153	}
154
155	free(impl_b_mode);
156	free(pipeCmd);
157	}
158	}
159	}
160
161	arb_assert(contradicted(fp, error));
162	arb_assert(implicated(error, error[0])); // deny empty error
163	return fp;
164	}
165
166	GB_ERROR ARB_zfclose(FILE *fp) {
167	bool fifo = GB_is_fifo(fp);
168
169	arb_assert(zfile_info.find(fp) != zfile_info.end()); // file was not opened using ARB_zfopen!
170
171	zinfo info = zfile_info[fp];
172	zfile_info.erase(fp);
173
174	int res;
175	if (fifo) {
176	res = pclose(fp);
177	}
178	else {
179	res = fclose(fp);
180	}
181
182	GB_ERROR error = NULp;
183	if (res != 0) {
184	int exited = WIFEXITED(res);
185	int status = WEXITSTATUS(res);
186	#if defined(DEBUG)
187	int signaled = WIFSIGNALED(res);
188	#endif
189
190	if (exited) {
191	if (status) {
192	if (fifo) {
193	error = GBS_global_string("pipe %s\n"
194	" file='%s'\n"
195	" using cmd='%s'\n"
196	" failed with exitcode=%i (broken pipe? corrupted archive?)\n",
197	info.isOutputPipe() ? "writing to" : "reading from",
198	info.get_filename(),
199	info.get_pipecmd(),
200	status);
201	}
202	}
203	}
204	if (!error) error = GB_IO_error("closing", info.get_filename());
205	#if defined(DEBUG)
206	error = GBS_global_string("%s (res=%i, exited=%i, signaled=%i, status=%i)", error, res, exited, signaled, status);
207	#endif
208	}
209	return error;
210	}
211
212	// --------------------------------------------------------------------------------
213
214	#ifdef UNIT_TESTS
215	#ifndef TEST_UNIT_H
216	#include <test_unit.h>
217	#endif
218
219	static char fileContent(FILE in, size_t& bytes_read) {
220	const size_t BUFFERSIZE = 1000;
221	char *buffer = ARB_alloc<char>(BUFFERSIZE+1);
222	bytes_read = fread(buffer, 1, BUFFERSIZE, in);
223	arb_assert(bytes_read<BUFFERSIZE);
224	buffer[bytes_read] = 0;
225	return buffer;
226	}
227
228	#define TEST_EXPECT_ZFOPEN_FAILS(name,mode,cmode,errpart) do{ \
229	GB_ERROR error = NULp; \
230	FILE *fp = ARB_zfopen(name, mode, cmode, error, false); \
231	\
232	if (fp) { \
233	TEST_EXPECT_NULL(error); \
234	error = ARB_zfclose(fp); \
235	} \
236	else { \
237	TEST_EXPECT_NULL(fp); \
238	} \
239	TEST_REJECT_NULL(error); \
240	TEST_EXPECT_CONTAINS(error, errpart); \
241	}while(0)
242
243	void TEST_compressed_io() {
244	const char *inText = "general/text.input";
245	const char *outFile = "compressed.out";
246
247	TEST_EXPECT_ZFOPEN_FAILS("", "", ZFILE_UNCOMPRESSED, "Invalid argument");
248	TEST_EXPECT_ZFOPEN_FAILS(outFile, "a", ZFILE_UNCOMPRESSED, "Cannot append to file using ARB_zfopen");
249	TEST_EXPECT_ZFOPEN_FAILS(outFile, "r", ZFILE_UNDEFINED, "Invalid compression mode");
250	TEST_EXPECT_ZFOPEN_FAILS(outFile, "w", ZFILE_AUTODETECT, "only works for input files");
251	TEST_EXPECT_ZFOPEN_FAILS(outFile, "rt", ZFILE_AUTODETECT, "Cannot use textmode");
252	TEST_EXPECT_ZFOPEN_FAILS(outFile, "r+", ZFILE_AUTODETECT, "Cannot open file in read and write mode");
253
254	#if defined(USE_BROKEN_COMPRESSION)
255	TEST_EXPECT_ZFOPEN_FAILS(outFile, "r", ZFILE_BROKEN, "broken pipe");
256	TEST_EXPECT_ZFOPEN_FAILS(outFile, "w", ZFILE_BROKEN, "broken pipe");
257	#endif
258
259	char *testText;
260	const size_t TEST_TEXT_SIZE = 428;
261	{
262	GB_ERROR error = NULp;
263	FILE *in = ARB_zfopen(inText, "r", ZFILE_UNCOMPRESSED, error, false);
264	TEST_EXPECT_NULL(error);
265	TEST_REJECT_NULL(in);
266
267	size_t bytes_read;
268	testText = fileContent(in, bytes_read);
269	TEST_EXPECT_EQUAL(bytes_read, TEST_TEXT_SIZE);
270
271	TEST_EXPECT_NO_ERROR(ARB_zfclose(in));
272	}
273
274	int successful_compressions = 0;
275
276	for (FileCompressionMode cmode = FileCompressionMode(ZFILE_AUTODETECT+1);
277	cmode != ZFILE_UNDEFINED;
278	cmode = FileCompressionMode(cmode+1))
279	{
280	TEST_ANNOTATE(GBS_global_string("cmode=%i", int(cmode)));
281
282	bool compressed_save_failed = false;
283	{
284	GB_ERROR error = NULp;
285	FILE *out = ARB_zfopen(outFile, "w", cmode, error, false);
286
287	TEST_EXPECT_NO_ERROR(error);
288	TEST_REJECT_NULL(out);
289
290	TEST_EXPECT_DIFFERENT(EOF, fputs(testText, out));
291
292	error = ARB_zfclose(out);
293	if (error && strstr(error, "failed with exitcode=127") && cmode != ZFILE_UNCOMPRESSED) {
294	// assume compression utility is not installed
295	compressed_save_failed = true;
296	}
297	else {
298	TEST_EXPECT_NO_ERROR(error);
299	}
300	}
301
302	if (!compressed_save_failed) {
303	for (int detect = 0; detect<=1; ++detect) {
304	TEST_ANNOTATE(GBS_global_string("cmode=%i detect=%i", int(cmode), detect));
305
306	GB_ERROR error = NULp;
307	FILE *in = ARB_zfopen(outFile, "r", detect ? ZFILE_AUTODETECT : cmode, error, false);
308
309	TEST_REJECT(error);
310	TEST_REJECT_NULL(in);
311
312	size_t bytes_read;
313	char *content = fileContent(in, bytes_read);
314	TEST_EXPECT_NO_ERROR(ARB_zfclose(in));
315	TEST_EXPECT_EQUAL(content, testText); // if this fails for detect==1 -> detection does not work
316	free(content);
317	}
318	successful_compressions++;
319	}
320	}
321
322	TEST_EXPECT(successful_compressions>=3); // at least ZFILE_UNCOMPRESSED, ZFILE_GZIP and ZFILE_BZIP should succeed
323
324	free(testText);
325	TEST_EXPECT_DIFFERENT(GB_unlink(outFile), -1);
326	}
327
328	#endif // UNIT_TESTS
329
330	// --------------------------------------------------------------------------------
331

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: trunk/CORE/arb_zfile.cxx

Download in other formats: