Context Navigation

source: tags/ms_r16q2/CORE/arb_zfile.cxx

Visit:

Last change on this file was 14583, checked in by westram, 8 years ago
NDEBUG warning
File size: 11.5 KB

Line
1	// ================================================================ //
2	// //
3	// File : arb_zfile.cxx //
4	// Purpose : Compressed file I/O //
5	// //
6	// Coded by Ralf Westram (coder@reallysoft.de) in November 2015 //
7	// http://www.arb-home.de/ //
8	// //
9	// ================================================================ //
10
11	#include "arb_zfile.h"
12	#include "arb_file.h"
13	#include "arb_msg.h"
14	#include "arb_misc.h"
15	#include "arb_assert.h"
16
17	#include <string>
18	#include <map>
19
20	using namespace std;
21
22	class zinfo {
23	// info stored for each sucessfully opened file
24	// to support proper error message on close.
25	bool writing; // false -> reading
26	string filename;
27	string pipe_cmd;
28	public:
29	zinfo() {}
30	zinfo(bool writing_, const char filename_, const char pipe_cmd_)
31	: writing(writing_),
32	filename(filename_),
33	pipe_cmd(pipe_cmd_)
34	{}
35
36	bool isOutputPipe() const { return writing; }
37	const char *get_filename() const { return filename.c_str(); }
38	const char *get_pipecmd() const { return pipe_cmd.c_str(); }
39	};
40	static map<FILE*,zinfo> zfile_info;
41
42	FILE ARB_zfopen(const char name, const char *mode, FileCompressionMode cmode, GB_ERROR& error, bool hideStderr) {
43	arb_assert(!error);
44
45	if (strchr(mode, 'a')) {
46	error = "Cannot append to file using ARB_zfopen";
47	return NULL;
48	}
49	if (strchr(mode, 't')) {
50	error = "Cannot use textmode for ARB_zfopen";
51	return NULL;
52	}
53	if (strchr(mode, '+')) {
54	error = "Cannot open file in read and write mode with ARB_zfopen";
55	return NULL;
56	}
57
58	bool forOutput = strchr(mode, 'w');
59	FILE *fp = NULL;
60
61	if (cmode == ZFILE_AUTODETECT) {
62	if (forOutput) {
63	error = "Autodetecting compression mode only works for input files";
64	}
65	else {
66	fp = fopen(name, "rb");
67	if (!fp) error = GB_IO_error("opening", name);
68	else {
69	// detect compression and set 'cmode'
70	const size_t MAGICSIZE = 5;
71	char buffer[MAGICSIZE];
72
73	size_t bytes_read = fread(buffer, 1, MAGICSIZE, fp);
74	fclose(fp);
75	fp = NULL;
76
77	if (bytes_read>=2 && strncmp(buffer, "\x1f\x8b", 2) == 0) cmode = ZFILE_GZIP;
78	else if (bytes_read>=2 && strncmp(buffer, "BZ", 2) == 0) cmode = ZFILE_BZIP2;
79	else if (bytes_read>=5 && strncmp(buffer, "\xfd" "7zXZ", 5) == 0) cmode = ZFILE_XZ;
80	else {
81	cmode = ZFILE_UNCOMPRESSED;
82	}
83	}
84	}
85	}
86
87	if (cmode == ZFILE_UNCOMPRESSED) {
88	fp = fopen(name, mode);
89	if (!fp) error = GB_IO_error("opening", name);
90	else {
91	zfile_info[fp] = zinfo(forOutput, name, "");
92	}
93	}
94	else {
95	if (!error) {
96	const char *compressor = NULL; // command used to compress (and decompress)
97	const char *decompress_flag = "-d"; // flag needed to decompress (assumes none to compress)
98
99	switch (cmode) {
100	case ZFILE_GZIP: {
101	static char *pigz = ARB_executable("pigz", ARB_getenv_ignore_empty("PATH"));
102	compressor = pigz ? pigz : "gzip";
103	break;
104	}
105	case ZFILE_BZIP2: compressor = "bzip2"; break;
106	case ZFILE_XZ: compressor = "xz"; break;
107
108	default:
109	error = GBS_global_string("Invalid compression mode (%i)", int(cmode));
110	break;
111
112	#if defined(USE_BROKEN_COMPRESSION)
113	case ZFILE_BROKEN:
114	compressor = "arb_weirdo"; // a non-existing command!
115	break;
116	#endif
117	}
118
119	if (!error) {
120	char *pipeCmd = forOutput
121	? GBS_global_string_copy("%s > %s", compressor, name)
122	: GBS_global_string_copy("%s %s < %s", compressor, decompress_flag, name);
123
124	if (hideStderr) {
125	freeset(pipeCmd, GBS_global_string_copy("( %s 2>/dev/null )", pipeCmd));
126	}
127
128	// remove 'b' from mode (pipes are binary by default)
129	char *impl_b_mode = strdup(mode);
130	while (1) {
131	char *b = strchr(impl_b_mode, 'b');
132	if (!b) break;
133	strcpy(b, b+1);
134	}
135
136	if (forOutput) { // write to pipe
137	fp = popen(pipeCmd, impl_b_mode);
138	if (!fp) error = GB_IO_error("writing to pipe", pipeCmd);
139	}
140	else { // read from pipe
141	fp = popen(pipeCmd, impl_b_mode);
142	if (!fp) error = GB_IO_error("reading from pipe", pipeCmd);
143	}
144
145	if (!error) {
146	zfile_info[fp] = zinfo(forOutput, name, pipeCmd);
147	}
148
149	free(impl_b_mode);
150	free(pipeCmd);
151	}
152	}
153	}
154
155	arb_assert(contradicted(fp, error));
156	arb_assert(implicated(error, error[0])); // deny empty error
157	return fp;
158	}
159
160	GB_ERROR ARB_zfclose(FILE *fp) {
161	bool fifo = GB_is_fifo(fp);
162	zinfo info = zfile_info[fp];
163	zfile_info.erase(fp);
164
165	int res;
166	if (fifo) {
167	res = pclose(fp);
168	}
169	else {
170	res = fclose(fp);
171	}
172
173	GB_ERROR error = NULL;
174	if (res != 0) {
175	int exited = WIFEXITED(res);
176	int status = WEXITSTATUS(res);
177	#if defined(DEBUG)
178	int signaled = WIFSIGNALED(res);
179	#endif
180
181	if (exited) {
182	if (status) {
183	if (fifo) {
184	error = GBS_global_string("pipe %s\n"
185	" file='%s'\n"
186	" using cmd='%s'\n"
187	" failed with exitcode=%i (broken pipe? corrupted archive?)\n",
188	info.isOutputPipe() ? "writing to" : "reading from",
189	info.get_filename(),
190	info.get_pipecmd(),
191	status);
192	}
193	}
194	}
195	if (!error) error = GB_IO_error("closing", info.get_filename());
196	#if defined(DEBUG)
197	error = GBS_global_string("%s (res=%i, exited=%i, signaled=%i, status=%i)", error, res, exited, signaled, status);
198	#endif
199	}
200	return error;
201	}
202
203	// --------------------------------------------------------------------------------
204
205	#ifdef UNIT_TESTS
206	#ifndef TEST_UNIT_H
207	#include <test_unit.h>
208	#endif
209
210	static char fileContent(FILE in, size_t& bytes_read) {
211	const size_t BUFFERSIZE = 1000;
212	char buffer = (char)malloc(BUFFERSIZE+1);
213	bytes_read = fread(buffer, 1, BUFFERSIZE, in);
214	arb_assert(bytes_read<BUFFERSIZE);
215	buffer[bytes_read] = 0;
216	return buffer;
217	}
218
219	#define TEST_EXPECT_ZFOPEN_FAILS(name,mode,cmode,errpart) do{ \
220	GB_ERROR error = NULL; \
221	FILE *fp = ARB_zfopen(name, mode, cmode, error, false); \
222	\
223	if (fp) { \
224	TEST_EXPECT_NULL(error); \
225	error = ARB_zfclose(fp); \
226	} \
227	else { \
228	TEST_EXPECT_NULL(fp); \
229	} \
230	TEST_REJECT_NULL(error); \
231	TEST_EXPECT_CONTAINS(error, errpart); \
232	}while(0)
233
234	void TEST_compressed_io() {
235	const char *inText = "general/text.input";
236	const char *outFile = "compressed.out";
237
238	TEST_EXPECT_ZFOPEN_FAILS("", "", ZFILE_UNCOMPRESSED, "Invalid argument");
239	TEST_EXPECT_ZFOPEN_FAILS(outFile, "a", ZFILE_UNCOMPRESSED, "Cannot append to file using ARB_zfopen");
240	TEST_EXPECT_ZFOPEN_FAILS(outFile, "r", ZFILE_UNDEFINED, "Invalid compression mode");
241	TEST_EXPECT_ZFOPEN_FAILS(outFile, "w", ZFILE_AUTODETECT, "only works for input files");
242	TEST_EXPECT_ZFOPEN_FAILS(outFile, "rt", ZFILE_AUTODETECT, "Cannot use textmode");
243	TEST_EXPECT_ZFOPEN_FAILS(outFile, "r+", ZFILE_AUTODETECT, "Cannot open file in read and write mode");
244
245	#if defined(USE_BROKEN_COMPRESSION)
246	TEST_EXPECT_ZFOPEN_FAILS(outFile, "r", ZFILE_BROKEN, "broken pipe");
247	TEST_EXPECT_ZFOPEN_FAILS(outFile, "w", ZFILE_BROKEN, "broken pipe");
248	#endif
249
250	char *testText;
251	const size_t TEST_TEXT_SIZE = 428;
252	{
253	GB_ERROR error = NULL;
254	FILE *in = ARB_zfopen(inText, "r", ZFILE_UNCOMPRESSED, error, false);
255	TEST_EXPECT_NULL(error);
256	TEST_REJECT_NULL(in);
257
258	size_t bytes_read;
259	testText = fileContent(in, bytes_read);
260	TEST_EXPECT_EQUAL(bytes_read, TEST_TEXT_SIZE);
261
262	TEST_EXPECT_NO_ERROR(ARB_zfclose(in));
263	}
264
265	int successful_compressions = 0;
266
267	for (FileCompressionMode cmode = FileCompressionMode(ZFILE_AUTODETECT+1);
268	cmode != ZFILE_UNDEFINED;
269	cmode = FileCompressionMode(cmode+1))
270	{
271	TEST_ANNOTATE(GBS_global_string("cmode=%i", int(cmode)));
272
273	bool compressed_save_failed = false;
274	{
275	GB_ERROR error = NULL;
276	FILE *out = ARB_zfopen(outFile, "w", cmode, error, false);
277
278	TEST_EXPECT_NO_ERROR(error);
279	TEST_REJECT_NULL(out);
280
281	TEST_EXPECT_DIFFERENT(EOF, fputs(testText, out));
282
283	error = ARB_zfclose(out);
284	if (error && strstr(error, "failed with exitcode=127") && cmode != ZFILE_UNCOMPRESSED) {
285	// assume compression utility is not installed
286	compressed_save_failed = true;
287	}
288	else {
289	TEST_EXPECT_NO_ERROR(error);
290	}
291	}
292
293	if (!compressed_save_failed) {
294	for (int detect = 0; detect<=1; ++detect) {
295	TEST_ANNOTATE(GBS_global_string("cmode=%i detect=%i", int(cmode), detect));
296
297	GB_ERROR error = NULL;
298	FILE *in = ARB_zfopen(outFile, "r", detect ? ZFILE_AUTODETECT : cmode, error, false);
299
300	TEST_REJECT(error);
301	TEST_REJECT_NULL(in);
302
303	size_t bytes_read;
304	char *content = fileContent(in, bytes_read);
305	TEST_EXPECT_NO_ERROR(ARB_zfclose(in));
306	TEST_EXPECT_EQUAL(content, testText); // if this fails for detect==1 -> detection does not work
307	free(content);
308	}
309	successful_compressions++;
310	}
311	}
312
313	TEST_EXPECT(successful_compressions>=3); // at least ZFILE_UNCOMPRESSED, ZFILE_GZIP and ZFILE_BZIP should succeed
314
315	free(testText);
316	TEST_EXPECT_DIFFERENT(GB_unlink(outFile), -1);
317	}
318
319	#endif // UNIT_TESTS
320
321	// --------------------------------------------------------------------------------
322

Note: See TracBrowser for help on using the repository browser.

Download in other formats: