Context Navigation

source: branches/profile/PROBE/PT_main.cxx

Visit:

Last change on this file was 11539, checked in by epruesse, 10 years ago
fix assertion failure in probe_struct_global::cleanup() on error during startup psg.com_so is expected to be null on PT_exit
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 21.0 KB

Line
1	// =============================================================== //
2	// //
3	// File : PT_main.cxx //
4	// Purpose : //
5	// //
6	// Institute of Microbiology (Technical University Munich) //
7	// http://www.arb-home.de/ //
8	// //
9	// =============================================================== //
10
11	#include "probe.h"
12	#include <PT_server_prototypes.h>
13	#include "pt_prototypes.h"
14	#include "PT_mem.h"
15
16	#include <BI_basepos.hxx>
17
18	#include <arbdbt.h>
19	#include <arb_file.h>
20	#include <arb_defs.h>
21	#include <arb_sleep.h>
22	#include <servercntrl.h>
23	#include <server.h>
24	#include <client.h>
25	#include <struct_man.h>
26	#include <ut_valgrinded.h>
27	#include <ptclean.h>
28
29	#include <unistd.h>
30	#include <sys/stat.h>
31	#include <time.h>
32
33	#define MAX_TRY 10
34	#define TIME_OUT 10006060*24
35
36	struct probe_struct_global psg;
37
38	// globals of gene-pt-server
39	int gene_flag = 0;
40
41	PT_main *aisc_main;
42
43	static time_t startTime;
44
45	static gene_struct_list all_gene_structs; // stores all gene_structs
46	gene_struct_index_arb gene_struct_arb2internal; // sorted by arb species+gene name
47	gene_struct_index_internal gene_struct_internal2arb; // sorted by internal name
48
49	// ----------------------------------------------
50	// global data initialization / cleanup
51
52	void probe_statistic_struct::setup() {
53	cut_offs = 0;
54	single_node = 0;
55	short_node = 0;
56	long_node = 0;
57	longs = 0;
58	shorts = 0;
59	shorts2 = 0;
60	chars = 0;
61
62	#ifdef ARB_64
63	int_node = 0;
64	ints = 0;
65	ints2 = 0;
66	maxdiff = 0;
67	#endif
68	}
69
70	void probe_struct_global::setup() {
71	// init uninitialized data
72
73	gb_shell = NULL;
74	gb_main = NULL;
75
76	alignment_name = NULL;
77	namehash = NULL;
78
79	data_count = 0;
80	data = NULL;
81
82	max_size = 0;
83	char_count = 0;
84
85	reversed = 0;
86
87	pos_to_weight = NULL;
88
89	sort_by = 0;
90
91	main_probe = NULL;
92	server_name = NULL;
93	link = NULL;
94
95	main.clear();
96
97	com_so = NULL;
98	pt.p1 = NULL;
99
100	stat.setup();
101	}
102
103	void probe_struct_global::cleanup() {
104	if (gb_main) {
105	delete [] data;
106
107	GB_close(gb_main);
108	gb_main = NULL;
109	}
110
111	if (gb_shell) {
112	delete gb_shell;
113	gb_shell = NULL;
114	}
115
116	if (namehash) GBS_free_hash(namehash);
117
118	free(ecoli);
119	delete bi_ecoli;
120	delete [] pos_to_weight;
121	free(alignment_name);
122
123	pt_assert(!com_so);
124
125	setup();
126	}
127
128	Memory MEM;
129
130	static bool psg_initialized = false;
131	void PT_init_psg() {
132	pt_assert(!psg_initialized);
133	psg.setup();
134	psg_initialized = true;
135	}
136
137	void PT_exit_psg() {
138	pt_assert(psg_initialized);
139	if (psg_initialized) {
140	psg.cleanup();
141	psg_initialized = false;
142	}
143	MEM.clear();
144	}
145
146	static void PT_exit() {
147	// unique exit point to ensure cleanup
148	if (aisc_main) destroy_PT_main(aisc_main);
149	if (psg_initialized) PT_exit_psg();
150	}
151
152	// ----------------------
153	// Communication
154
155	static ARB_ERROR pt_init_main_struct(PT_main , const char filename) { // __ATTR__USERESULT
156	ARB_ERROR error = probe_read_data_base(filename, true);
157	if (!error) {
158	GB_transaction ta(psg.gb_main);
159	psg.alignment_name = GBT_get_default_alignment(psg.gb_main);
160	if (!psg.alignment_name && GB_have_error()) error = GB_await_error();
161	}
162
163	if (!error) {
164	printf("Building PT-Server for alignment '%s'...\n", psg.alignment_name);
165	error = PT_init_input_data();
166	PT_build_species_hash();
167	}
168	return error;
169	}
170
171	int server_shutdown(PT_main /pm*/, aisc_string passwd) {
172	// password check
173	if (strcmp(passwd, "47@#34543df43%&3667gh")) return 1;
174
175	fflush_all();
176	fprintf(stderr, "\nARB_PT_SERVER: received shutdown message\n");
177
178	// shutdown clients
179	aisc_broadcast(psg.com_so, 0, "Used PT-server has been shut down");
180
181	// shutdown server
182	aisc_server_shutdown(psg.com_so);
183	PT_exit();
184	fflush_all();
185	exit(EXIT_SUCCESS);
186	}
187
188	int broadcast(PT_main *main, int) {
189	aisc_broadcast(psg.com_so, main->m_type, main->m_text);
190	return 0;
191	}
192
193	// ------------------------------------------------------------------------------ name mapping
194	// the mapping is generated in ../TOOLS/arb_gene_probe.cxx
195
196	inline const char find_sep(const char str, char sep) {
197	// sep may occur escaped (by \)
198	const char *found = strchr(str, sep);
199	while (found) {
200	if (found>str && found[-1] == '\\') { // escaped separator
201	found = strchr(found+1, sep);
202	}
203	else {
204	break; // non-escaped -> report
205	}
206	}
207	return found;
208	}
209
210	inline bool copy_to_buf(const char start, const char behindEnd, int MAXLEN, char *destBuf) {
211	int len = behindEnd-start;
212	if (len>MAXLEN) {
213	return false;
214	}
215	memcpy(destBuf, start, len);
216	destBuf[len] = 0;
217	return true;
218	}
219
220	__ATTR__USERESULT static ARB_ERROR parse_names_into_gene_struct(const char *map_str, gene_struct_list& listOfGenes) {
221	#define MAX_INAME_LEN 30
222	#define MAX_ONAME_LEN 30
223	#define MAX_GNAME_LEN 1024
224
225	char iname[MAX_INAME_LEN+1]; // internal name
226	char oname[MAX_ONAME_LEN+1]; // organism name
227	char gname[MAX_GNAME_LEN+1]; // gene name
228
229	ARB_ERROR err;
230
231	while (*map_str) {
232	const char *sep1 = strchr(map_str, ';');
233	const char *sep2 = sep1 ? strchr(sep1+1, ';') : 0;
234	const char *sep3 = sep2 ? find_sep(sep2+1, ';') : 0;
235
236	if (sep3) {
237	bool ok = copy_to_buf(map_str, sep1, MAX_INAME_LEN, iname);
238	ok = ok && copy_to_buf(sep1+1, sep2, MAX_ONAME_LEN, oname);
239	ok = ok && copy_to_buf(sep2+1, sep3, MAX_GNAME_LEN, gname);
240
241	if (ok) {
242	char *unesc = GBS_unescape_string(gname, ";", '\\');
243	listOfGenes.push_back(gene_struct(iname, oname, unesc));
244	free(unesc);
245	}
246	else {
247	err = "buffer overflow (some name too long)";
248	break;
249	}
250	}
251	else {
252	err = GBS_global_string("expected at least 3 ';' in '%s'", map_str);
253	break;
254	}
255	map_str = sep3+1;
256	}
257
258	if (err) err = GBS_global_string("while parsing name-mapping: %s", err.deliver());
259
260	return err;
261	}
262
263	static GB_ERROR PT_init_map() { // goes to header: __ATTR__USERESULT
264	GB_ERROR error = GB_push_transaction(psg.gb_main);
265	if (!error) {
266	GBDATA *gb_gene_map = GB_entry(psg.gb_main, "gene_map");
267
268	if (gb_gene_map) {
269	gene_flag = 1;
270
271	GBDATA *map_ptr_str = GB_entry(gb_gene_map, "map_string");
272	const char *map_str = GB_read_char_pntr(map_ptr_str);
273
274	error = parse_names_into_gene_struct(map_str, all_gene_structs).deliver();
275
276	// build indices :
277	gene_struct_list::const_iterator end = all_gene_structs.end();
278
279	for (gene_struct_list::const_iterator gs = all_gene_structs.begin(); gs != end; ++gs) {
280	if (gene_struct_internal2arb.find(&*gs) != gene_struct_internal2arb.end()) {
281	fprintf(stderr, " Duplicated internal entry for '%s'\n", gs->get_internal_gene_name());
282	}
283	gene_struct_internal2arb.insert(&*gs);
284	if (gene_struct_arb2internal.find(&*gs) != gene_struct_arb2internal.end()) {
285	fprintf(stderr, " Duplicated entry for '%s/%s'\n", gs->get_arb_species_name(), gs->get_arb_gene_name());
286	}
287	gene_struct_arb2internal.insert(&*gs);
288	}
289
290	size_t list_size = all_gene_structs.size();
291	if (list_size == 0) {
292	error = "name map is empty";
293	}
294	else if (gene_struct_arb2internal.size() != list_size) {
295	size_t dups = list_size-gene_struct_arb2internal.size();
296	error = GBS_global_string("detected %zi duplicated 'species/gene' combinations in name mapping", dups);
297	}
298	}
299	else {
300	gene_flag = 0;
301	}
302	}
303
304	return GB_end_transaction(psg.gb_main, error);
305	}
306
307	__ATTR__USERESULT static ARB_ERROR start_pt_server(const char socket_name, const char arbdb_name, const char pt_name, const char exename) {
308	ARB_ERROR error;
309
310	fprintf(stdout,
311	"\n"
312	"ARB POS_TREE SERVER v%i (C)1993-2013 by O.Strunk, J.Boehnel, R.Westram\n"
313	"initializing:\n"
314	"- opening connection...\n",
315	PT_SERVER_VERSION);
316	GB_sleep(1, SEC);
317
318	Hs_struct *so = NULL;
319	{
320	const int MAX_STARTUP_SEC = 100;
321	const int RETRY_AFTER_SEC = 5;
322	for (int i = 0; i<MAX_STARTUP_SEC; i += RETRY_AFTER_SEC) {
323	so = open_aisc_server(socket_name, TIME_OUT, 0);
324	if (so) break;
325
326	printf(" Cannot bind to socket (retry in %i seconds)\n", RETRY_AFTER_SEC);
327	GB_sleep(RETRY_AFTER_SEC, SEC);
328	}
329	}
330
331	if (!so) error = "can't bind to socket";
332	else {
333	psg.com_so = so;
334
335	struct stat arbdb_stat;
336	if (stat(arbdb_name, &arbdb_stat)) {
337	error = GB_IO_error("stat", arbdb_name);
338	error = GBS_global_string("Source DB missing (%s)", error.deliver());
339	}
340	if (!error) {
341	const char *update_reason = NULL;
342
343	{
344	struct stat pt_stat;
345	if (stat(pt_name, &pt_stat)) {
346	update_reason = GB_IO_error("stat", pt_name);
347	}
348	else if (arbdb_stat.st_mtime > pt_stat.st_mtime) {
349	update_reason = GBS_global_string("'%s' has been modified more recently than '%s'", arbdb_name, pt_name);
350	}
351	else if (pt_stat.st_size == 0) {
352	update_reason = GBS_global_string("'%s' is empty", pt_name);
353	}
354	}
355
356	if (update_reason) {
357	printf("- updating postree (Reason: %s)\n", update_reason);
358
359	// run build_clean
360	char *cmd = GBS_global_string_copy("%s -build_clean -D%s", exename, arbdb_name);
361	make_valgrinded_call(cmd);
362	error = GBK_system(cmd);
363	free(cmd);
364
365	// run build
366	if (!error) {
367	cmd = GBS_global_string_copy("%s -build -D%s", exename, arbdb_name);
368	make_valgrinded_call(cmd);
369	error = GBK_system(cmd);
370	free(cmd);
371	}
372
373	if (error) error = GBS_global_string("Failed to update postree (Reason: %s)", error.deliver());
374	}
375	}
376	if (!error) {
377	// init server main struct
378	fputs("- init internal structs...\n", stdout);
379	{
380	void *reserved_for_mmap = NULL;
381	long size_of_file = GB_size_of_file(pt_name);
382	if (size_of_file > 0) {
383	reserved_for_mmap = malloc(size_of_file);
384	if (!reserved_for_mmap) {
385	error = GBS_global_string("cannot reserve enough memory to map postree (needed=%li)", size_of_file);
386	}
387	}
388
389	if (!error) error = pt_init_main_struct(aisc_main, arbdb_name);
390	free(reserved_for_mmap);
391	}
392
393	if (!error) error = enter_stage_2_load_tree(aisc_main, pt_name);
394	if (!error) error = PT_init_map();
395
396	if (!error) {
397	// all ok -> main "loop"
398	{
399	time_t now; time(&now);
400	fflush_all();
401	printf("[startup took %s]\n", GBS_readable_timediff(difftime(now, startTime)));
402	}
403
404	printf("ok, server is running.\n"); // do NOT change or remove! others depend on it
405
406	fflush_all();
407	aisc_accept_calls(so);
408	}
409	}
410	aisc_server_shutdown(psg.com_so);
411	}
412	return error;
413	}
414
415	static int get_DB_state(GBDATA *gb_main, ARB_ERROR& error) {
416	pt_assert(!error);
417	error = GB_push_transaction(gb_main);
418	int state = -1;
419	if (!error) {
420	GBDATA *gb_ptserver = GBT_find_or_create(gb_main, "ptserver", 7);
421	long *statePtr = gb_ptserver ? GBT_readOrCreate_int(gb_ptserver, "dbstate", 0) : NULL;
422	if (!statePtr) {
423	error = GB_await_error();
424	}
425	else {
426	state = *statePtr;
427	}
428	}
429	error = GB_end_transaction(gb_main, error);
430	return state;
431	}
432
433	static GB_ERROR set_DB_state(GBDATA *gb_main, int dbstate) {
434	GB_ERROR error = GB_push_transaction(gb_main);
435	if (!error) {
436	GBDATA *gb_ptserver = GBT_find_or_create(gb_main, "ptserver", 7);
437	GBDATA *gb_state = gb_ptserver ? GB_searchOrCreate_int(gb_ptserver, "dbstate", 0) : NULL;
438	if (!gb_state) error = GB_await_error();
439	else error = GB_write_int(gb_state, dbstate);
440	}
441	error = GB_end_transaction(gb_main, error);
442	return error;
443	}
444
445	__ATTR__USERESULT static ARB_ERROR run_command(const char exename, const char command, const arb_params *params) {
446	ARB_ERROR error;
447	char *msg = NULL;
448
449	// check that arb_pt_server knows its socket
450	const char *socket_name = params->tcp;
451	if (!socket_name) {
452	socket_name = GBS_read_arb_tcp("ARB_PT_SERVER0");
453	if (!socket_name) {
454	error = GB_await_error();
455	error = GBS_global_string("Don't know which socket to use (Reason: %s)", error.deliver());
456	}
457	}
458
459	if (!error) {
460	char *pt_name = GBS_global_string_copy("%s.pt", params->db_server);
461
462	if (strcmp(command, "-build_clean") == 0) { // cleanup source DB
463	error = probe_read_data_base(params->db_server, false);
464	if (!error) {
465	pt_assert(psg.gb_main);
466
467	error = GB_no_transaction(psg.gb_main); // don't waste memory for transaction (no abort may happen)
468
469	if (!error) {
470	int dbstate = get_DB_state(psg.gb_main, error);
471	if (!error && dbstate>0) {
472	if (dbstate == 1) {
473	fputs("Warning: database already has been prepared for ptserver\n", stdout);
474	}
475	else {
476	error = GBS_global_string("unexpected dbstate='%i'", dbstate);
477	}
478	}
479
480	if (!error && dbstate == 0) {
481	GB_push_my_security(psg.gb_main);
482
483	if (!error) error = cleanup_ptserver_database(psg.gb_main, PTSERVER);
484	if (!error) error = PT_prepare_data(psg.gb_main);
485
486	if (!error) error = set_DB_state(psg.gb_main, 1);
487
488	GB_pop_my_security(psg.gb_main);
489
490	if (!error) {
491	const char *mode = GB_supports_mapfile() ? "bfm" : "bf";
492	error = GB_save_as(psg.gb_main, params->db_server, mode);
493	}
494	}
495	}
496	}
497	}
498	else if (strcmp(command, "-build") == 0) { // build command
499	if (!error) error = pt_init_main_struct(aisc_main, params->db_server);
500	if (!error) {
501	int dbstate = get_DB_state(psg.gb_main, error);
502	if (!error && dbstate != 1) {
503	error = "database has not been prepared for ptserver";
504	}
505	}
506	if (error) error = GBS_global_string("Gave up (Reason: %s)", error.deliver());
507
508	if (!error) {
509	ULONG ARM_size_kb = 0;
510	{
511	const char *mapfile = GB_mapfile(psg.gb_main);
512	if (GB_is_regularfile(mapfile)) {
513	ARM_size_kb = GB_size_of_file(mapfile)/1024.0+0.5;
514	}
515	else {
516	fputs("Warning: cannot detect size of mapfile (have none).\n"
517	" Ptserver will probably use too much memory.\n",
518	stderr);
519	}
520	}
521
522	error = enter_stage_1_build_tree(aisc_main, pt_name, ARM_size_kb);
523	if (error) {
524	error = GBS_global_string("Failed to build index (Reason: %s)", error.deliver());
525	}
526	else {
527	msg = GBS_global_string_copy("PT_SERVER database \"%s\" has been created.", params->db_server);
528	}
529	}
530	}
531	else if (strcmp(command, "-QUERY") == 0) {
532	error = pt_init_main_struct(aisc_main, params->db_server);
533	if (error) error = GBS_global_string("Gave up (Reason: %s)", error.deliver());
534	else {
535	error = enter_stage_2_load_tree(aisc_main, pt_name); // now stage 2
536	#if defined(CALCULATE_STATS_ON_QUERY)
537	if (!error) {
538	puts("[index loaded - calculating statistic]");
539	PT_dump_tree_statistics(pt_name);
540	puts("[statistic done]");
541	}
542	#endif
543	}
544	}
545	else {
546	GB_ERROR openerr = NULL;
547	psg.link = aisc_open(socket_name, psg.main, AISC_MAGIC_NUMBER, &openerr);
548
549	if (openerr) {
550	error = openerr;
551	}
552	else {
553	bool running = psg.link;
554	bool kill = false;
555	bool start = false;
556
557	if (strcmp(command, "-look") == 0) { start = !running; }
558	else if (strcmp(command, "-boot") == 0) { kill = running; start = true; }
559	else if (strcmp(command, "-kill") == 0) { kill = running; }
560	else { error = GBS_global_string("Unknown command '%s'", command); }
561
562	if (!error) {
563	if (kill) {
564	pt_assert(running);
565	fputs("There is another active server. Sending shutdown message..\n", stderr);
566	if (aisc_nput(psg.link, PT_MAIN, psg.main, MAIN_SHUTDOWN, "47@#34543df43%&3667gh", NULL)) {
567	fprintf(stderr,
568	"%s: Warning: Problem connecting to the running %s\n"
569	" You might need to kill it manually to ensure proper operation\n",
570	exename, exename);
571	}
572	aisc_close(psg.link, psg.main);
573	psg.link = 0;
574	}
575
576	if (start) {
577	error = start_pt_server(socket_name, params->db_server, pt_name, exename); // @@@ does not always return - fix! (see also server_shutdown())
578	}
579	}
580	}
581	}
582
583	free(pt_name);
584	}
585
586	if (error) msg = strdup(error.preserve());
587	if (msg) {
588	puts(msg);
589	GBS_add_ptserver_logentry(msg);
590
591	char *quoted_msg = GBS_string_eval(msg, ":'=\"", 0);
592	pt_assert(quoted_msg);
593	char *msg_command = GBS_global_string_copy("arb_message '%s' &", quoted_msg);
594
595	if (system(msg_command) != 0) fprintf(stderr, "Failed to run '%s'\n", msg_command);
596
597	free(msg_command);
598	free(quoted_msg);
599
600	free(msg);
601	}
602
603	return error;
604	}
605
606	int ARB_main(int argc, char *argv[]) {
607	time(&startTime);
608
609	int exitcode = EXIT_SUCCESS;
610	arb_params params = arb_trace_argv(&argc, (const char *)argv);
611	const char *exename = argv[0];
612
613	PT_init_psg();
614	GB_install_pid(0); // not arb_clean able
615
616	extern int aisc_core_on_error;
617	aisc_core_on_error = 0;
618
619	// parse arguments
620	char *command = NULL;
621	{
622	if ((argc>2) \|\|
623	((argc<2) && !params->db_server) \|\|
624	(argc >= 2 && strcmp(argv[1], "--help") == 0))
625	{
626	fprintf(stderr, "Syntax: %s [-look/-build/-kill/-QUERY/-boot] -Dfile.arb -TSocketid\n", exename);
627	exitcode = EXIT_FAILURE;
628	}
629	else {
630	if (argc==2) command = strdup(argv[1]);
631	else command = strdup("-boot");
632	}
633	}
634
635	if (!command) {
636	fprintf(stderr, "%s: Error: No command specified on command line", exename);
637	}
638	else {
639	aisc_main = create_PT_main();
640
641	GB_ERROR error = run_command(exename, command, params).deliver();
642	if (error) {
643	fprintf(stderr, "%s: Error: %s\n", exename, error);
644	exitcode = EXIT_FAILURE;
645	}
646	else {
647	time_t now; time(&now);
648	fflush_all();
649	printf("[ptserver '%s' took %s]\n", command, GBS_readable_timediff(difftime(now, startTime)));
650	fflush(stdout);
651	}
652	free(command);
653	}
654
655	free_arb_params(params);
656	PT_exit();
657	fflush_all();
658	return exitcode;
659	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: