Context Navigation

source: tags/svn.1.5.4/NTREE/NT_dbrepair.cxx

Visit:

Last change on this file was 8355, checked in by westram, 13 years ago
moved aw_question() and relatives into aw_question.hxx
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 40.8 KB

Line
1	// =============================================================== //
2	// //
3	// File : NT_dbrepair.cxx //
4	// Purpose : repair database bugs //
5	// //
6	// Coded by Ralf Westram (coder@reallysoft.de) in May 2008 //
7	// Institute of Microbiology (Technical University Munich) //
8	// http://www.arb-home.de/ //
9	// //
10	// =============================================================== //
11
12	#include "NT_dbrepair.hxx"
13
14	#include <arbdbt.h>
15	#include <adGene.h>
16
17	#include <items.h>
18	#include <GEN.hxx>
19	#include <EXP.hxx>
20	#include <aw_color_groups.hxx>
21	#include <aw_msg.hxx>
22	#include <arb_progress.h>
23	#include <aw_question.hxx>
24
25	#include <arb_str.h>
26	#include <arb_strarray.h>
27
28	#include <map>
29	#include <set>
30	#include <string>
31	#include <vector>
32
33
34	#define nt_assert(bed) arb_assert(bed)
35
36	using namespace std;
37
38	#if defined(WARN_TODO)
39	#warning the whole fix mechanism should be part of some lower-level-library
40	// meanwhile DB checks are only performed by ARB_NTREE
41	// ItemSelector should go to same library as this module
42	#endif
43
44	// --------------------------------------------------------------------------------
45	// CheckedConsistencies provides an easy way to automatically correct flues in the database
46	// by calling a check routine exactly once.
47	//
48	// For an example see nt_check_database_consistency()
49	//
50	// Note: this makes problems if DB is loaded with older ARB version and some already
51	// fixed flues a put into DB again.
52	// see http://bugs.arb-home.de/ticket/143
53
54	typedef GB_ERROR (item_check_fun)(GBDATA gb_item, ItemSelector& sel);
55
56	typedef map<string, item_check_fun> item_check_map;
57	typedef item_check_map::const_iterator item_check_iter;
58
59	class CheckedConsistencies : virtual Noncopyable {
60	GBDATA *gb_main;
61	size_t species_count;
62	size_t sai_count;
63	set<string> consistencies;
64	item_check_map item_checks;
65
66	GB_ERROR perform_selected_item_checks(ItemSelector& sel);
67
68	public:
69
70	CheckedConsistencies(GBDATA *gb_main_) : gb_main(gb_main_) {
71	GB_transaction ta(gb_main);
72	GBDATA *gb_checks = GB_search(gb_main, "checks", GB_CREATE_CONTAINER);
73
74	for (GBDATA *gb_check = GB_entry(gb_checks, "check"); gb_check; gb_check = GB_nextEntry(gb_check)) {
75	consistencies.insert(GB_read_char_pntr(gb_check));
76	}
77
78	species_count = GBT_get_species_count(gb_main);
79	sai_count = GBT_get_SAI_count(gb_main);
80	}
81
82	bool was_performed(const string& check_name) const {
83	return consistencies.find(check_name) != consistencies.end();
84	}
85
86	GB_ERROR register_as_performed(const string& check_name) {
87	GB_ERROR error = 0;
88	if (was_performed(check_name)) {
89	printf("check '%s' already has been registered before. Duplicated check name?\n", check_name.c_str());
90	}
91	else {
92	GB_transaction ta(gb_main);
93
94	GBDATA *gb_checks = GB_search(gb_main, "checks", GB_CREATE_CONTAINER);
95	GBDATA *gb_check = GB_create(gb_checks, "check", GB_STRING);
96
97	if (!gb_check) error = GB_await_error();
98	else error = GB_write_string(gb_check, check_name.c_str());
99
100	if (!error) consistencies.insert(check_name);
101	}
102	return error;
103	}
104
105	void perform_check(const string& check_name,
106	GB_ERROR (do_check)(GBDATA gb_main, size_t species, size_t sais),
107	GB_ERROR& error)
108	{
109	if (!error && !was_performed(check_name)) {
110	arb_progress progress(check_name.c_str());
111	error = do_check(gb_main, species_count, sai_count);
112	if (!error) register_as_performed(check_name);
113	}
114	}
115
116	void register_item_check(const string& check_name, item_check_fun item_check) {
117	if (!was_performed(check_name)) {
118	item_checks[check_name] = item_check;
119	}
120	}
121
122	void perform_item_checks(GB_ERROR& error);
123
124	GB_ERROR forgetDoneChecks() {
125	GB_ERROR error = 0;
126	GB_transaction ta(gb_main);
127
128	GBDATA *gb_checks = GB_search(gb_main, "checks", GB_CREATE_CONTAINER);
129	for (GBDATA *gb_check = GB_entry(gb_checks, "check"); gb_check && !error; gb_check = GB_nextEntry(gb_check)) {
130	char *check_name = GB_read_string(gb_check);
131
132	#if defined(DEBUG)
133	printf("Deleting check '%s'\n", check_name);
134	#endif // DEBUG
135	error = GB_delete(gb_check);
136	consistencies.erase(check_name);
137	free(check_name);
138	}
139	return error;
140	}
141	};
142
143	GB_ERROR CheckedConsistencies::perform_selected_item_checks(ItemSelector& sel) {
144	GB_ERROR error = NULL;
145	item_check_iter end = item_checks.end();
146
147	for (GBDATA *gb_cont = sel.get_first_item_container(gb_main, NULL, QUERY_ALL_ITEMS);
148	gb_cont && !error;
149	gb_cont = sel.get_next_item_container(gb_cont, QUERY_ALL_ITEMS))
150	{
151	for (GBDATA *gb_item = sel.get_first_item(gb_cont, QUERY_ALL_ITEMS);
152	gb_item && !error;
153	gb_item = sel.get_next_item(gb_item, QUERY_ALL_ITEMS))
154	{
155	for (item_check_iter chk = item_checks.begin(); chk != end && !error; ++chk) {
156	error = chk->second(gb_item, sel);
157	}
158	}
159	}
160
161	return error;
162	}
163
164	void CheckedConsistencies::perform_item_checks(GB_ERROR& error) {
165	if (!item_checks.empty()) {
166	if (!error) {
167	GB_transaction ta(gb_main);
168	bool is_genome_db = GEN_is_genome_db(gb_main, -1);
169
170	error = perform_selected_item_checks(SPECIES_get_selector());
171	if (!error && is_genome_db) {
172	error = perform_selected_item_checks(GEN_get_selector());
173	if (!error) error = perform_selected_item_checks(EXP_get_selector());
174	}
175
176	error = ta.close(error);
177	}
178
179	if (!error) {
180	item_check_iter end = item_checks.end();
181	for (item_check_iter chk = item_checks.begin(); chk != end && !error; ++chk) {
182	error = register_as_performed(chk->first);
183	}
184
185	if (!error) item_checks.clear();
186	}
187	}
188	}
189
190	// --------------------------------------------------------------------------------
191
192	static GB_ERROR NT_fix_gene_data(GBDATA gb_main, size_t species_count, size_t / sai_count */) {
193	GB_transaction ta(gb_main);
194	arb_progress progress(species_count);
195
196	size_t deleted_gene_datas = 0;
197	size_t generated_gene_datas = 0;
198	GB_ERROR error = 0;
199
200	for (GBDATA *gb_species = GBT_first_species(gb_main);
201	gb_species && !error;
202	gb_species = GBT_next_species(gb_species))
203	{
204	bool is_organism = (GB_entry(gb_species, GENOM_ALIGNMENT) != 0); // same test as GEN_is_organism, but w/o genome-db-assertion
205	GBDATA *gb_gene_data = GEN_find_gene_data(gb_species);
206
207	if (is_organism && !gb_gene_data) {
208	gb_gene_data = GEN_findOrCreate_gene_data(gb_species);
209	generated_gene_datas++;
210	}
211	else if (!is_organism && gb_gene_data) {
212	GBDATA *gb_child = GB_child(gb_gene_data);
213	if (!gb_child) {
214	error = GB_delete(gb_gene_data);
215	if (!error) deleted_gene_datas++;
216	}
217	else {
218	error = GBS_global_string("Non-empty 'gene_data' found for species '%s',\n"
219	"which has no alignment '" GENOM_ALIGNMENT "',\n"
220	"i.e. which is not regarded as full-genome organism.\n"
221	"This causes problems - please fix!",
222	GBT_read_name(gb_species));
223	}
224	}
225
226	progress.inc_and_check_user_abort(error);
227	}
228
229	if (!error) {
230	if (deleted_gene_datas) {
231	aw_message(GBS_global_string("Deleted %zu useless empty 'gene_data' entries.", deleted_gene_datas));
232	}
233	if (generated_gene_datas) {
234	aw_message(GBS_global_string("Re-created %zu missing 'gene_data' entries.\nThese organisms have no genes yet!", generated_gene_datas));
235	}
236	}
237	return ta.close(error);
238	}
239
240	// --------------------------------------------------------------------------------
241
242	static GBDATA expectField(GBDATA gb_gene, const char *field, GB_ERROR& data_error) {
243	GBDATA *gb_field = 0;
244	if (!data_error) {
245	gb_field = GB_entry(gb_gene, field);
246	if (!gb_field) data_error = GBS_global_string("Expected field '%s' missing", field);
247	}
248	return gb_field;
249	}
250
251	static GBDATA disexpectField(GBDATA gb_gene, const char *field, GB_ERROR& data_error) {
252	GBDATA *gb_field = 0;
253	if (!data_error) {
254	gb_field = GB_entry(gb_gene, field);
255	if (gb_field) data_error = GBS_global_string("Unexpected field '%s' exists (wrong value in pos_joined?)", field);
256	}
257	GBS_reuse_buffer(field);
258	return gb_field;
259	}
260
261	static GB_ERROR NT_convert_gene_locations(GBDATA gb_main, size_t species_count, size_t / sai_count */) {
262	GB_transaction ta(gb_main);
263	GB_ERROR error = 0;
264	long fixed_genes = 0;
265	long skipped_genes = 0;
266	long genes = 0;
267
268	typedef vector<GBDATA*> GBvec;
269	GBvec toDelete;
270
271	arb_progress progress(species_count);
272
273	for (GBDATA *gb_organism = GEN_first_organism(gb_main);
274	gb_organism && !error;
275	gb_organism = GEN_next_organism(gb_organism))
276	{
277	GBDATA *gb_gene_data = GEN_find_gene_data(gb_organism);
278	nt_assert(gb_gene_data);
279	if (gb_gene_data) {
280	for (GBDATA *gb_gene = GEN_first_gene_rel_gene_data(gb_gene_data);
281	gb_gene && !error;
282	gb_gene = GEN_next_gene(gb_gene))
283	{
284	genes++;
285
286	int parts = 1;
287	{
288	GBDATA *gb_pos_joined = GB_entry(gb_gene, "pos_joined");
289	if (gb_pos_joined) parts = GB_read_int(gb_pos_joined); // its a joined gene
290	}
291
292	GBDATA *gb_pos_start = GB_entry(gb_gene, "pos_start"); // test for new format
293	if (!gb_pos_start) {
294	GBDATA *gb_pos_begin = GB_entry(gb_gene, "pos_begin"); // test for old format
295	if (!gb_pos_begin) {
296	error = "Neither 'pos_begin' nor 'pos_start' found - format of gene location is unknown";
297	}
298	}
299
300	if (!gb_pos_start && !error) { // assume old format
301	// parts<-1 would be valid in new format, but here we have old format
302	if (parts<1) error = GBS_global_string("Illegal value in 'pos_joined' (%i)", parts);
303
304	GB_ERROR data_error = 0; // error in this gene -> don't convert
305	GEN_position *pos = GEN_new_position(parts, false); // all were joinable (no information about it was stored)
306
307	// parse old gene information into 'pos'
308	//
309	// old-format was:
310	// Start-Positions: pos_begin, pos_begin2, pos_begin3, ...
311	// End-Positions: pos_end, pos_end2, pos_end3, ...
312	// Joined?: pos_joined (always >= 1)
313	// Complement: complement (one entry for all parts)
314	// Certainty: pos_uncertain (maybe pos_uncertain1 etc.)
315
316	int complement = 0;
317	{
318	GBDATA *gb_complement = GB_entry(gb_gene, "complement");
319	if (gb_complement) {
320	complement = GB_read_byte(gb_complement);
321	toDelete.push_back(gb_complement);
322	}
323	}
324
325	bool has_uncertain_fields = false;
326	for (int p = 1; p <= parts && !error && !data_error; ++p) {
327	GBDATA *gb_pos_begin = 0;
328	GBDATA *gb_pos_end = 0;
329	const char *pos_uncertain_field = 0;
330
331	if (p == 1) {
332	gb_pos_begin = expectField(gb_gene, "pos_begin", data_error);
333	gb_pos_end = expectField(gb_gene, "pos_end", data_error);
334
335	pos_uncertain_field = "pos_uncertain";
336	}
337	else {
338	const char *pos_begin_field = GBS_global_string("pos_begin%i", p);
339	const char *pos_end_field = GBS_global_string("pos_end%i", p);
340
341	gb_pos_begin = expectField(gb_gene, pos_begin_field, data_error);
342	gb_pos_end = expectField(gb_gene, pos_end_field, data_error);
343
344	GBS_reuse_buffer(pos_end_field);
345	GBS_reuse_buffer(pos_begin_field);
346
347	if (!data_error) pos_uncertain_field = GBS_global_string("pos_uncertain%i", p);
348	}
349
350	int pospos = complement ? (parts-p) : (p-1);
351
352	if (!data_error) {
353	GBDATA *gb_pos_uncertain = GB_entry(gb_gene, pos_uncertain_field);
354
355	if (!gb_pos_uncertain) {
356	if (has_uncertain_fields) data_error = GBS_global_string("Expected field '%s' missing", pos_uncertain_field);
357	}
358	else {
359	if (p == 1) has_uncertain_fields = true;
360	else {
361	if (!has_uncertain_fields) {
362	data_error = GBS_global_string("Found '%s' as first certainty-information", pos_uncertain_field);
363	}
364	}
365	}
366
367	if (!data_error) {
368	int begin = GB_read_int(gb_pos_begin);
369	int end = GB_read_int(gb_pos_end);
370
371	pos->start_pos[pospos] = begin;
372	pos->stop_pos[pospos] = end;
373	pos->complement[pospos] = complement; // set all complement entries to same value (old format only had one complement entry)
374
375	if (gb_pos_uncertain) {
376	const char *uncertain = GB_read_char_pntr(gb_pos_uncertain);
377
378	if (!uncertain) error = GB_await_error();
379	else {
380	if (!pos->start_uncertain) GEN_use_uncertainties(pos);
381
382	if (strlen(uncertain) != 2) {
383	data_error = "wrong length";
384	}
385	else {
386	for (int up = 0; up<2; up++) {
387	if (strchr("<=>", uncertain[up]) == 0) {
388	data_error = GBS_global_string("illegal character '%c'", uncertain[up]);
389	}
390	else {
391	(up == 0 ? pos->start_uncertain[pospos] : pos->stop_uncertain[pospos]) = uncertain[up];
392	}
393	}
394	}
395
396
397	toDelete.push_back(gb_pos_uncertain);
398	}
399	}
400
401	toDelete.push_back(gb_pos_begin);
402	toDelete.push_back(gb_pos_end);
403	}
404	}
405	}
406
407	for (int p = parts+1; p <= parts+4 && !error && !data_error; ++p) {
408	disexpectField(gb_gene, GBS_global_string("pos_begin%i", p), data_error);
409	disexpectField(gb_gene, GBS_global_string("pos_end%i", p), data_error);
410	disexpectField(gb_gene, GBS_global_string("complement%i", p), data_error);
411	disexpectField(gb_gene, GBS_global_string("pos_uncertain%i", p), data_error);
412	}
413
414	// now save new position data
415
416	if (data_error) {
417	skipped_genes++;
418	}
419	else if (!error) {
420	error = GEN_write_position(gb_gene, pos);
421
422	if (!error) {
423	// delete old-format entries
424	GBvec::const_iterator end = toDelete.end();
425	for (GBvec::const_iterator i = toDelete.begin(); i != end && !error; ++i) {
426	error = GB_delete(*i);
427	}
428
429	if (!error) fixed_genes++;
430	}
431	}
432
433	toDelete.clear();
434	GEN_free_position(pos);
435
436	if (data_error \|\| error) {
437	char *gene_id = GEN_global_gene_identifier(gb_gene, gb_organism);
438	if (error) {
439	error = GBS_global_string("Gene '%s': %s", gene_id, error);
440	}
441	else {
442	aw_message(GBS_global_string("Gene '%s' was not converted, fix data manually!\nReason: %s", gene_id, data_error));
443	}
444	free(gene_id);
445	}
446	}
447	}
448	}
449
450	progress.inc_and_check_user_abort(error);
451	}
452
453	if (!error) {
454	if (fixed_genes>0) aw_message(GBS_global_string("Fixed location entries of %li genes.", fixed_genes));
455	if (skipped_genes>0) {
456	aw_message(GBS_global_string("Didn't fix location entries of %li genes (see warnings).", skipped_genes));
457	error = "Not all gene locations were fixed.\nFix manually, save DB and restart ARB with that DB.\nMake sure you have a backup copy of the original DB!";
458	}
459
460	if (fixed_genes \|\| skipped_genes) {
461	long already_fixed_genes = genes-(fixed_genes+skipped_genes);
462	if (already_fixed_genes>0) aw_message(GBS_global_string("Location entries of %li genes already were in new format.", already_fixed_genes));
463	}
464	}
465
466	return error;
467	}
468
469
470	// --------------------------------------------------------------------------------
471
472	static GB_ERROR NT_del_mark_move_REF(GBDATA *gb_main, size_t species_count, size_t sai_count) {
473	GB_transaction ta(gb_main);
474	GB_ERROR error = 0;
475	size_t all = species_count+sai_count;
476	size_t removed = 0;
477
478	// delete 'mark' entries from all alignments of species/SAIs
479
480	arb_progress progress(all);
481	ConstStrArray ali_names;
482	GBT_get_alignment_names(ali_names, gb_main);
483
484	for (int pass = 0; pass < 2 && !error; ++pass) {
485	for (GBDATA *gb_item = (pass == 0) ? GBT_first_species(gb_main) : GBT_first_SAI(gb_main);
486	gb_item && !error;
487	gb_item = (pass == 0) ? GBT_next_species(gb_item) : GBT_next_SAI(gb_item))
488	{
489	for (int ali = 0; ali_names[ali] && !error; ++ali) {
490	GBDATA *gb_ali = GB_entry(gb_item, ali_names[ali]);
491	if (gb_ali) {
492	GBDATA *gb_mark = GB_entry(gb_ali, "mark");
493	if (gb_mark) {
494	error = GB_delete(gb_mark);
495	removed++;
496	}
497	}
498	}
499
500	progress.inc_and_check_user_abort(error);
501	}
502	}
503
504	{
505	char *helix_name = GBT_get_default_helix(gb_main);
506	GBDATA *gb_helix = GBT_find_SAI(gb_main, helix_name);
507
508	if (gb_helix) {
509	for (int ali = 0; ali_names[ali] && !error; ++ali) {
510	GBDATA *gb_ali = GB_entry(gb_helix, ali_names[ali]);
511	GBDATA *gb_old_ref = GB_entry(gb_ali, "REF");
512	GBDATA *gb_new_ref = GB_entry(gb_ali, "_REF");
513
514	if (gb_old_ref) {
515	if (gb_new_ref) {
516	error = GBS_global_string("SAI:%s has 'REF' and '_REF' in '%s' (data corrupt?!)",
517	helix_name, ali_names[ali]);
518	}
519	else { // move info from REF -> _REF
520	char *content = GB_read_string(gb_old_ref);
521	if (!content) error = GB_await_error();
522	else {
523	gb_new_ref = GB_create(gb_ali, "_REF", GB_STRING);
524	if (!gb_new_ref) error = GB_await_error();
525	else {
526	error = GB_write_string(gb_new_ref, content);
527	if (!error) error = GB_delete(gb_old_ref);
528	}
529	free(content);
530	}
531	}
532	}
533	}
534	}
535
536	free(helix_name);
537	}
538
539	if (!error) {
540	if (removed) {
541	aw_message(GBS_global_string("Deleted %zu useless 'mark' entries.", removed));
542	}
543	}
544
545	return ta.close(error);
546	}
547
548	// --------------------------------------------------------------------------------
549
550	static bool testDictionaryCompression(GBDATA *gbd, GBQUARK key_quark, bool testUse) {
551	// returns true, if
552	// testUse == true and ANY entries below 'gbd' with quark 'key_quark' uses dictionary compression
553	// testUse == false and ALL entries below 'gbd' with quark 'key_quark' can be decompressed w/o errors
554
555	nt_assert(GB_read_type(gbd) == GB_DB);
556
557	for (GBDATA *gb_sub = GB_child(gbd); gb_sub; gb_sub = GB_nextChild(gb_sub)) {
558	switch (GB_read_type(gb_sub)) {
559	case GB_DB:
560	// return false if any compression failed or return true if any uses dict-compression
561	if (testDictionaryCompression(gb_sub, key_quark, testUse) == testUse) return testUse;
562	break;
563
564	case GB_STRING:
565	case GB_LINK:
566	if (GB_get_quark(gb_sub) == key_quark && GB_is_dictionary_compressed(gb_sub)) {
567	if (testUse) return true;
568
569	const char *decompressed = GB_read_char_pntr(gb_sub);
570	if (!decompressed) return false;
571	}
572	break;
573
574	default:
575	break;
576	}
577	}
578
579	return !testUse;
580	}
581
582	class Dict;
583	typedef SmartPtr<Dict> DictPtr;
584
585
586	class KeyInfo : virtual Noncopyable {
587	string name; // keyname
588	DictPtr original;
589
590	bool compressionTested;
591	bool compressed;
592
593	void init() {
594	compressionTested = false;
595	compressed = false;
596	}
597
598	public:
599	KeyInfo(const char *Name) : name(Name) { init(); }
600	KeyInfo(const char *Name, DictPtr originalDict) : name(Name), original(originalDict) { init(); }
601
602	void testCompressed(GBDATA *gb_main) {
603	nt_assert(!compressionTested);
604	compressed = testDictionaryCompression(gb_main, GB_key_2_quark(gb_main, name.c_str()), true);
605	compressionTested = true;
606	}
607
608	const string& getName() const { return name; }
609
610	bool isCompressed() const {
611	nt_assert(compressionTested);
612	return compressed;
613	}
614	};
615
616
617	class Dict : virtual Noncopyable {
618	string group; // lowercase keyname
619	string orgkey;
620	DictData *data;
621
622	map<string, bool> decompressWorks; // key -> bool
623
624	public:
625	static GBDATA *gb_main;
626
627	Dict(const char Group, const char OrgKey, DictData *Data) : group(Group), orgkey(OrgKey), data(Data) {}
628
629	const string& getGroup() const { return group; }
630	const string& getOriginalKey() const { return orgkey; }
631
632	bool mayBeUsedWith(const string& key) const { return strcasecmp(group.c_str(), key.c_str()) == 0; }
633
634	GB_ERROR assignToKey(const string& key) const { return GB_set_dictionary(gb_main, key.c_str(), data); }
635	GB_ERROR unassignFromKey(const string& key) const { return GB_set_dictionary(gb_main, key.c_str(), NULL); }
636
637	bool canDecompress(const string& key) {
638	nt_assert(mayBeUsedWith(key));
639	if (decompressWorks.find(key) == decompressWorks.end()) {
640	bool works = false;
641	GB_ERROR error = assignToKey(key);
642
643	if (!error) works = testDictionaryCompression(gb_main, GB_key_2_quark(gb_main, key.c_str()), false);
644	decompressWorks[key] = works;
645
646	GB_ERROR err2 = unassignFromKey(key);
647	if (err2) {
648	aw_message(GBS_global_string("Error while removing @dictionary from key '%s': %s", key.c_str(), err2));
649	}
650	}
651	return decompressWorks[key];
652	}
653	};
654	GBDATA *Dict::gb_main = NULL;
655
656
657	typedef map<string, int> KeyCounter; // groupname -> occur count
658	typedef SmartPtr<KeyInfo> KeyInfoPtr;
659	typedef map<string, KeyInfoPtr> Keys; // keyname -> info
660	typedef map<string, DictPtr> DictMap;
661	typedef vector<DictPtr> Dicts;
662	typedef set<string> StringSet;
663
664	#define STATUS_PREFIX "Dictionary: "
665
666	template<typename CONT, typename KEY>
667	bool contains(const CONT& container, const KEY& key) {
668	return container.find(key) != container.end();
669	}
670
671	static GB_ERROR findAffectedKeys(GBDATA *gb_key_data, KeyCounter& kcount, Keys& keys, Dicts& dicts) {
672	GB_ERROR error = 0;
673	GBDATA *gb_main = GB_get_root(gb_key_data);
674
675	for (int pass = 1; pass <= 2; ++pass) {
676	for (GBDATA *gb_key = GB_entry(gb_key_data, "@key"); !error && gb_key; gb_key = GB_nextEntry(gb_key)) {
677	GBDATA *gb_name = GB_entry(gb_key, "@name");
678	const char *keyName = GB_read_char_pntr(gb_name);
679
680	if (!keyName) {
681	error = GBS_global_string("@key w/o @name (%s)", GB_await_error());
682	}
683	else {
684	char *keyGroup = strdup(keyName);
685	ARB_strlower(keyGroup);
686
687	switch (pass) {
688	case 1:
689	kcount[keyGroup]++;
690	break;
691	case 2:
692	if (kcount[keyGroup]>1) {
693	GBDATA *gb_dictionary = GB_entry(gb_key, "@dictionary");
694	if (gb_dictionary) {
695	DictPtr dict = new Dict(keyGroup, keyName, GB_get_dictionary(gb_main, keyName));
696	keys[keyName] = new KeyInfo(keyName, dict);
697	dicts.push_back(dict);
698	}
699	else keys[keyName] = new KeyInfo(keyName);
700	}
701	else kcount.erase(keyGroup);
702	break;
703	}
704	free(keyGroup);
705	}
706	}
707	}
708	return error;
709	}
710
711	static GB_ERROR deleteDataOfKey(GBDATA *gbd, GBQUARK key_quark, StringSet& deletedData, long& deleted, long& notDeleted) {
712	GB_ERROR error = 0;
713	for (GBDATA *gb_sub = GB_child(gbd); gb_sub; gb_sub = GB_nextChild(gb_sub)) {
714	switch (GB_read_type(gb_sub)) {
715	case GB_DB:
716	error = deleteDataOfKey(gb_sub, key_quark, deletedData, deleted, notDeleted);
717	break;
718
719	case GB_STRING:
720	case GB_LINK:
721	if (GB_get_quark(gb_sub) == key_quark) {
722	if (GB_is_dictionary_compressed(gb_sub)) {
723	string path(GB_get_db_path(gb_sub));
724	error = GB_delete(gb_sub);
725	if (!error) {
726	deletedData.insert(path);
727	deleted++;
728	}
729	}
730	else {
731	notDeleted++;
732	}
733	}
734	break;
735	default:
736	break;
737	}
738	}
739	return error;
740	}
741
742	static char readFirstCompressedDataOf(GBDATA gbd, GBQUARK key_quark) {
743	char *data = 0;
744	for (GBDATA *gb_sub = GB_child(gbd); !data && gb_sub; gb_sub = GB_nextChild(gb_sub)) {
745	switch (GB_read_type(gb_sub)) {
746	case GB_DB:
747	data = readFirstCompressedDataOf(gb_sub, key_quark);
748	break;
749
750	case GB_STRING:
751	case GB_LINK:
752	if (GB_get_quark(gb_sub) == key_quark) {
753	if (GB_is_dictionary_compressed(gb_sub)) {
754	data = GB_read_as_string(gb_sub);
755	}
756	}
757	break;
758	default:
759	break;
760	}
761	}
762	return data;
763	}
764
765
766	static GB_ERROR NT_fix_dict_compress(GBDATA *gb_main, size_t, size_t) {
767	GB_transaction ta(gb_main);
768	GBDATA *gb_key_data = GB_search(gb_main, "__SYSTEM__/@key_data", GB_FIND);
769	GB_ERROR error = 0;
770
771	Dict::gb_main = gb_main;
772
773	if (!gb_key_data) {
774	error = "No @key_data found.. DB corrupted?";
775	}
776	else {
777	KeyCounter kcount; // strlwr(keyname) -> count
778	Keys keys;
779	Dicts dicts;
780
781	error = findAffectedKeys(gb_key_data, kcount, keys, dicts);
782
783	// count affected keys
784	int affectedKeys = 0;
785	for (KeyCounter::iterator kci = kcount.begin(); kci != kcount.end(); ++kci) {
786	affectedKeys += kci->second;
787	}
788
789	if (!error && affectedKeys>0) {
790	// check which keys are compressed
791
792	{
793	arb_progress progress(STATUS_PREFIX "search compressed data", affectedKeys);
794
795	for (Keys::iterator ki = keys.begin(); ki != keys.end(); ++ki) {
796	KeyInfoPtr k = ki->second;
797	k->testCompressed(gb_main);
798	++progress;
799	}
800	}
801
802	// test which key/dict combinations work
803	int combinations = 0; // possible key/dict combinations
804
805	DictMap use; // keyname -> dictionary (which dictionary to use)
806	StringSet multiDecompressible; // keys which can be decompressed with multiple dictionaries
807
808	for (int pass = 1; pass <= 2; ++pass) {
809	arb_progress *progress = NULL;
810	if (pass == 2 && combinations) progress = new arb_progress(STATUS_PREFIX "test compression", combinations);
811
812	for (Dicts::iterator di = dicts.begin(); di != dicts.end(); ++di) {
813	DictPtr d = *di;
814
815	for (Keys::iterator ki = keys.begin(); ki != keys.end(); ++ki) {
816	KeyInfoPtr k = ki->second;
817	const string& keyname = k->getName();
818
819	if (k->isCompressed() && d->mayBeUsedWith(keyname)) {
820	switch (pass) {
821	case 1:
822	combinations++;
823	break;
824	case 2:
825	if (d->canDecompress(keyname)) {
826	if (!contains(use, keyname)) { // first dictionary working with keyname
827	use[keyname] = d;
828	}
829	else { // already have another dictionary working with keyname
830	multiDecompressible.insert(keyname);
831	}
832	}
833	++(*progress);
834	break;
835	}
836	}
837	}
838	}
839	delete progress;
840	}
841
842	StringSet notDecompressible; // keys which can be decompressed with none of the dictionaries
843	for (Keys::iterator ki = keys.begin(); ki != keys.end(); ++ki) {
844	KeyInfoPtr k = ki->second;
845	const string& keyname = k->getName();
846
847	if (k->isCompressed()) {
848	if (!contains(use, keyname)) notDecompressible.insert(keyname);
849	if (contains(multiDecompressible, keyname)) use.erase(keyname);
850	}
851	}
852
853	bool dataLost = false;
854	int reassigned = 0;
855
856	if (!notDecompressible.empty()) {
857	// bad .. found undecompressible data
858	int nd_count = notDecompressible.size();
859	aw_message(GBS_global_string("Detected corrupted dictionary compression\n"
860	"Data of %i DB-keys is lost and will be deleted", nd_count));
861
862	arb_progress progress(STATUS_PREFIX "deleting corrupt data", nd_count);
863
864	StringSet deletedData;
865	long deleted = 0;
866	long notDeleted = 0;
867
868	for (StringSet::iterator ki = notDecompressible.begin(); !error && ki != notDecompressible.end(); ++ki) {
869	const string& keyname = *ki;
870
871	error = deleteDataOfKey(gb_main, GB_key_2_quark(gb_main, keyname.c_str()), deletedData, deleted, notDeleted);
872	++progress;
873	}
874
875	if (!error) {
876	nt_assert(deleted); // at least 1 db-entry should have been deleted
877
878	aw_message(GBS_global_string("Deleted %li of %li affected DB-entries", deleted, deleted+notDeleted));
879	aw_message("see console for a list of affected keys");
880
881	printf("Deleted keys:\n");
882	for (StringSet::iterator di = deletedData.begin(); di != deletedData.end(); ++di) {
883	printf("* %s\n", di->c_str());
884	}
885	}
886	}
887
888	if (!error && !multiDecompressible.empty()) {
889	for (StringSet::iterator ki = multiDecompressible.begin(); !error && ki != multiDecompressible.end(); ++ki) {
890	const string& keyname = *ki;
891	int possible = 0;
892	vector<DictPtr> possibleDicts;
893
894	printf("--------------------------------------------------------------------------------\n");
895
896	for (Dicts::iterator di = dicts.begin(); !error && di != dicts.end(); ++di) {
897	DictPtr d = *di;
898	if (d->mayBeUsedWith(keyname) && d->canDecompress(keyname)) {
899	error = d->assignToKey(keyname);
900	if (!error) {
901	char *data = readFirstCompressedDataOf(gb_main, GB_key_2_quark(gb_main, keyname.c_str()));
902
903	nt_assert(data);
904	possible++;
905	printf("possibility %i = '%s'\n", possible, data);
906	free(data);
907
908	possibleDicts.push_back(d);
909
910	error = d->unassignFromKey(keyname);
911	}
912	}
913	}
914
915	if (!error) {
916	nt_assert(possible>0);
917
918	int selected;
919	if (possible>1) {
920	char *question = GBS_global_string_copy("%i possibilities to decompress field '%s' have been detected\n"
921	"and example data was dumped to the console.\n"
922	"Please examine output and decide which is the correct possibility!",
923	possible, keyname.c_str());
924
925	const char *buttons = "Abort";
926	for (int p = 1; p <= possible; ++p) buttons = GBS_global_string("%s,%i", buttons, p);
927	selected = aw_question(question, buttons, false, NULL);
928	free(question);
929	}
930	else {
931	selected = 1;
932	}
933
934	if (!selected) {
935	error = "Aborted by user";
936	}
937	else {
938	use[keyname] = possibleDicts[selected-1];
939	}
940	}
941	}
942	}
943
944	// now all redundancies should be eliminated and we can assign dictionaries to affected keys
945	if (!error) {
946	for (Keys::iterator ki = keys.begin(); !error && ki != keys.end(); ++ki) {
947	KeyInfoPtr k = ki->second;
948	const string& keyname = k->getName();
949
950	if (k->isCompressed()) {
951	if (!contains(use, keyname)) {
952	error = GBS_global_string("No dictionary detected for key '%s'", keyname.c_str());
953	}
954	else {
955	DictPtr d = use[keyname];
956
957	if (d->getOriginalKey() != keyname) {
958	d->assignToKey(keyname); // set the dictionary
959	aw_message(GBS_global_string("Assigning '%s'-dictionary to '%s'",
960	d->getOriginalKey().c_str(), keyname.c_str()));
961	reassigned++;
962	}
963	}
964	}
965	}
966	}
967
968	if (dataLost\|\|reassigned) {
969	aw_message(dataLost
970	? "We apologize for the data-loss."
971	: "No conflicts detected in compressed data.");
972	aw_message("Dictionaries fixed.\n"
973	"Please save your database with a new name.");
974	}
975	}
976	}
977
978	Dict::gb_main = NULL;
979	return ta.close(error);
980	}
981
982	// --------------------------------------------------------------------------------
983
984	static GB_ERROR remove_dup_colors(GBDATA *gb_item, ItemSelector& IF_DEBUG(sel)) {
985	// Databases out there may contain multiple 'ARB_color' entries.
986	// Due to some already fixed bug - maybe introduced in r5309 and fixed in r5825
987
988	GBDATA *gb_color = GB_entry(gb_item, AW_COLOR_GROUP_ENTRY);
989	GB_ERROR error = NULL;
990
991	#if defined(DEBUG)
992	int del_count = 0;
993	#endif // DEBUG
994
995	if (gb_color) {
996	GB_push_my_security(gb_color);
997	while (!error) {
998	GBDATA *gb_next_color = GB_nextEntry(gb_color);
999	if (!gb_next_color) break;
1000
1001	error = GB_delete(gb_next_color);
1002	#if defined(DEBUG)
1003	if (!error) del_count++;
1004	#endif // DEBUG
1005	}
1006	GB_pop_my_security(gb_color);
1007	}
1008
1009	#if defined(DEBUG)
1010	if (del_count) fprintf(stderr,
1011	"- deleted %i duplicated '" AW_COLOR_GROUP_ENTRY "' from %s '%s'\n",
1012	del_count,
1013	sel.item_name,
1014	sel.generate_item_id(GB_get_root(gb_item), gb_item));
1015	#endif // DEBUG
1016
1017	return error;
1018	}
1019
1020	// --------------------------------------------------------------------------------
1021
1022	GB_ERROR NT_repair_DB(GBDATA *gb_main) {
1023	// status is already open and will be closed by caller!
1024
1025	CheckedConsistencies check(gb_main);
1026	GB_ERROR err = 0;
1027	bool is_genome_db;
1028	{
1029	GB_transaction ta(gb_main);
1030	is_genome_db = GEN_is_genome_db(gb_main, -1);
1031	}
1032
1033	check.perform_check("fix gene_data", NT_fix_gene_data, err);
1034	check.perform_check("fix_dict_compress", NT_fix_dict_compress, err); // do this before NT_del_mark_move_REF (cause 'REF' is affected)
1035	check.perform_check("del_mark_move_REF", NT_del_mark_move_REF, err);
1036
1037	if (is_genome_db) {
1038	check.perform_check("convert_gene_locations", NT_convert_gene_locations, err);
1039	}
1040
1041	check.register_item_check("duplicated_item_colors", remove_dup_colors);
1042	check.perform_item_checks(err);
1043
1044	return err;
1045	}
1046
1047	void NT_rerepair_DB(AW_window*, AW_CL cl_gbmain, AW_CL) {
1048	// re-perform all DB checks
1049	GBDATA gb_main = reinterpret_cast<GBDATA>(cl_gbmain);
1050	GB_ERROR err = 0;
1051	{
1052	CheckedConsistencies check(gb_main);
1053	err = check.forgetDoneChecks();
1054	}
1055	if (!err) {
1056	arb_progress progress("DB-Repair");
1057	err = NT_repair_DB(gb_main);
1058	}
1059
1060	if (err) aw_message(err);
1061	}
1062
1063

Note: See TracBrowser for help on using the repository browser.

Download in other formats: