Context Navigation

source: branches/nameserver/SL/XFERSET/xferset.cxx

Visit:

Last change on this file was 18098, checked in by westram, 6 years ago
stuff leak in test code.
File size: 69.7 KB

Line
1	// ========================================================= //
2	// //
3	// File : xferset.c //
4	// Purpose : field transfer sets //
5	// //
6	// Coded by Ralf Westram (coder@reallysoft.de) in Mar 19 //
7	// http://www.arb-home.de/ //
8	// //
9	// ========================================================= //
10
11	#include "xferset.h"
12
13	#include <ConfigMapping.h>
14	#include <BufferedFileReader.h>
15	#include <arbdbt.h>
16	#include <arb_str.h>
17	#include <arb_stdstr.h>
18
19	#include <set>
20
21	using namespace std;
22
23	namespace FieldTransfer {
24
25	typedef set<string, NoCaseCmp> StrSet;
26
27	static void StrSet2StrArray(const StrSet& src, StrArray& dst) {
28	for (StrSet::const_iterator i = src.begin(); i != src.end(); ++i) {
29	dst.put(strdup(i->c_str()));
30	}
31	}
32	static void StrArray2StrSet(const StrArray& src, StrSet& dst) {
33	for (unsigned i = 0; i<src.size(); ++i) {
34	dst.insert(src[i]);
35	}
36	}
37
38	void RuleSet::extractUsedFields(StrArray& input, StrArray& output) const { // @@@ want flavor just filling 2 StrSets
39	StrSet in, out;
40	for (unsigned i = 0; i<size(); ++i) {
41	const Rule& rule = get(i);
42	const string& srcFields = rule.getSourceFields();
43	if (rule.multiple_source_fields()) {
44	ConstStrArray ifield;
45	GBT_split_string(ifield, srcFields.c_str(), ';');
46	for (unsigned f = 0; f<ifield.size(); ++f) {
47	const char *source = ifield[f];
48	if (source[0]) in.insert(source);
49	}
50	}
51	else {
52	if (!srcFields.empty()) in.insert(srcFields);
53	}
54	const string& target = rule.targetField();
55	if (!target.empty()) out.insert(target);
56	}
57	StrSet2StrArray(in, input);
58	StrSet2StrArray(out, output);
59	}
60
61	TransportedData ReadRule::readTypedFromField(GB_TYPES readAsType, GBDATA *gb_field) const {
62	xf_assert(GB_read_type(gb_field) != GB_DB); // fails e.g. if rule defined for a name used by a container
63
64	switch (readAsType) {
65	case GB_INT: {
66	int asInt = GB_read_int(gb_field);
67	return TransportedData(asInt);
68	}
69	case GB_FLOAT: {
70	float asFloat = GB_read_float(gb_field);
71	return TransportedData(asFloat);
72	}
73	case GB_STRING: {
74	char *asStr = GB_read_as_string(gb_field);
75	string data(asStr);
76	free(asStr);
77	return TransportedData(data);
78	}
79	default: xf_assert(0); break; // invalid type
80	}
81	xf_assert(0); // should be never reached.
82	return TransportedData::none();
83	}
84
85	TransportedData ReadRule::aciAppliedTo(const string& toStr, GBDATA *gb_main) const {
86	// @@@ use environment instead of gb_main?
87	char *result = GB_command_interpreter(toStr.c_str(), aci.c_str(), gb_main);
88	if (result) {
89	string converted(result);
90	free(result);
91	return TransportedData(converted);
92	}
93	return TransportedData::makeError(GB_await_error());
94	}
95
96	inline TransportedData cannotReadContainer(const char *containerName) {
97	return TransportedData::makeError(GBS_global_string("cannot read as data ('%s' is a container)", containerName));
98	}
99
100	TransportedData ReadRule::readFrom(GBDATA *gb_item) const {
101	if (!gb_item) { // got no item -> can't read
102	return TransportedData::makeError("lacking item to readFrom");
103	}
104
105	if (fields.empty()) {
106	return TransportedData::makeError("no source field(s) specified");
107	}
108
109	if (multiple_source_fields()) {
110	ConstStrArray field;
111	GBT_split_string(field, fields.c_str(), ';');
112
113	string concat;
114	bool gotData = false; // at least one input field found?
115	for (size_t f = 0; f<field.size(); ++f) {
116	GBDATA *gb_field = GB_search(gb_item, field[f], GB_FIND);
117	if (gb_field) {
118	GB_TYPES sourceType = GB_read_type(gb_field);
119	if (sourceType == GB_DB) {
120	return cannotReadContainer(field[f]);
121	}
122
123	TransportedData plain = readTypedFromField(GB_STRING, gb_field); // ignores sourceType
124	if (plain.failed()) return plain;
125
126	xf_assert(plain.exists());
127	if (!concat.empty()) concat += separator;
128	concat += plain.getString();
129
130	gotData = true;
131	}
132	else if (GB_have_error()) {
133	return TransportedData::makeError(GB_await_error());
134	}
135	}
136
137	if (gotData) {
138	if (!aci.empty()) {
139	return aciAppliedTo(concat, GB_get_root(gb_item));
140	}
141	return TransportedData(concat);
142	}
143	// otherwise: do not transport if all source fields are missing
144	}
145	else {
146	GBDATA *gb_field = GB_search(gb_item, fields.c_str(), GB_FIND);
147	if (gb_field) {
148	GB_TYPES sourceType = GB_read_type(gb_field);
149	if (sourceType == GB_DB) {
150	return cannotReadContainer(fields.c_str());
151	}
152	if (!aci.empty()) {
153	TransportedData plain = readTypedFromField(GB_STRING, gb_field); // ignores sourceType
154	// @@@ store sourceType if no dest-type deduced?
155	return aciAppliedTo(plain.getString(), GB_get_root(gb_item));
156	}
157	return readTypedFromField(sourceType, gb_field);
158	}
159	else if (GB_have_error()) {
160	return TransportedData::makeError(GB_await_error());
161	}
162	// otherwise: do not transport if source field is missing
163	}
164	// if field does not exist -> report "no type"
165	return TransportedData::none();
166	}
167
168	static GB_ERROR unconvertedWrite(const TransportedData& data, GBDATA *gb_field) {
169	GB_ERROR error = NULp;
170	switch (data.getType()) {
171	case GB_STRING: {
172	const char *str = data.getString().c_str();
173	error = GB_write_string(gb_field, str);
174	break;
175	}
176	case GB_INT: {
177	int num = data.getInt();
178	error = GB_write_int(gb_field, num);
179	break;
180	}
181	case GB_FLOAT: {
182	float fnum = data.getFloat();
183	error = GB_write_float(gb_field, fnum);
184	break;
185	}
186	default: { // unhandled type
187	xf_assert(0);
188	break;
189	}
190	}
191	return error;
192	}
193	static GB_ERROR convertAndWrite(const TransportedData& data, GBDATA *gb_field, GB_TYPES wantedTargetType, bool acceptLossyConversion) {
194	// perform conversion to 'wantedTargetType' and write to 'gb_field'
195	GB_ERROR error = NULp;
196
197	switch (data.getType()) {
198	case GB_INT:
199	if (wantedTargetType == GB_FLOAT) {
200	// convert int -> float
201	float f = data.getInt();
202	int32_t i = int(f+.5); // round (just in case some underflow happened, causing sth like 4711.9999999)
203
204	if (i != data.getInt() && !acceptLossyConversion) {
205	error = GBS_global_string("lossy int->float type conversion (%i->%i)", data.getInt(), i);
206	}
207	else {
208	error = GB_write_float(gb_field, f);
209	}
210	}
211	else {
212	error = GB_write_lossless_int(gb_field, data.getInt());
213	}
214	break;
215
216	case GB_FLOAT:
217	if (wantedTargetType == GB_INT) {
218	// convert float -> int
219	double d = data.getFloat();
220	int i = d>0 ? (int)(d+0.5) : (int)(d-0.5);
221	// @@@ increment a round-counter in RuleSet?
222	double d2 = i;
223
224	if (d != d2 && !acceptLossyConversion) { // precision loss
225	error = GBS_global_string("lossy float->int type conversion (%e->%e)", d, d2);
226	}
227	else {
228	error = GB_write_int(gb_field, i);
229	}
230	}
231	else {
232	error = GB_write_lossless_float(gb_field, data.getFloat());
233	}
234	break;
235
236	case GB_STRING:
237	error = GB_write_autoconv_string(gb_field, data.getString().c_str()); // @@@ avoid silent data loss
238	// @@@ use GBT_write_float_converted / GBT_write_int_converted here!
239	break;
240
241	default:
242	xf_assert(0); // unhandled type
243	break;
244	}
245
246	return error;
247	}
248
249	GB_ERROR WriteRule::writeTo(const TransportedData& data, GBDATA *gb_item, bool acceptLossyConversion) const {
250	if (!gb_item) return "lacking item to writeTo";
251
252	// @@@ overwrite existing target field? should it be allowed or denied? optional?
253	// @@@ try GBT_searchOrCreate_itemfield_according_to_changekey to create a field
254	xf_assert(data.exists());
255
256	GB_TYPES usedTargetType = forcesType() ? getTargetType() : data.getType();
257
258	GB_ERROR error = check_hkey();
259	if (!error) {
260	GBDATA *gb_field = GB_search(gb_item, name.c_str(), usedTargetType); // Note: works with hierarchical keys
261	if (!gb_field) {
262	error = GB_await_error(); // field not created -> report why
263	}
264	else {
265	if (data.getType() == usedTargetType) { // data and target have same type -> no conversion needed
266	error = unconvertedWrite(data, gb_field);
267	}
268	else { // type differs -> perform conversion (act like field converter reachable from info-window)
269	error = convertAndWrite(data, gb_field, usedTargetType, acceptLossyConversion);
270	}
271	}
272	}
273	return error;
274	}
275
276	GB_ERROR Rule::transferBy(GBDATA gb_source, GBDATA gb_dest) const {
277	/! apply one rule (as part of transfer. /
278
279	// @@@ detect target field type. has to be done before starting transfer (do only once for each rule!)
280	// @@@ pass target field type to reader (to select best read method)!
281
282	GB_ERROR error = NULp;
283	TransportedData tdata = readFrom(gb_source);
284	if (tdata.failed()) {
285	error = tdata.getError();
286	}
287	else if (tdata.exists()) {
288	error = writeTo(tdata, gb_dest, precisionLossPermitted());
289	}
290	// else source missing -> do nothing.
291	// Note: if target field exists and source field is missing -> target field remains intact.
292
293	xf_assert(!GB_have_error()); // invalid to export an error (should get returned)
294	return error;
295	}
296
297	GB_ERROR RuleSet::transferBy(GBDATA gb_source, GBDATA gb_dest) const {
298	/! transfer field data by applying all rules. /
299
300	GB_ERROR error = NULp;
301	size_t r;
302	for (r = 0; r<size() && !error; ++r) {
303	const Rule& rule = get(r);
304	error = rule.transferBy(gb_source, gb_dest);
305	}
306	if (error) {
307	error = GBS_global_string("%s (in rule #%zu)", error, r);
308	}
309
310	xf_assert(!GB_have_error()); // invalid to export an error (should get returned)
311	return error;
312	}
313
314	GB_ERROR RuleSet::saveTo(const char *filename) const {
315	GB_ERROR error = NULp;
316	FILE *out = fopen(filename, "wt");
317	if (!out) {
318	error = GB_IO_error("saving", filename);
319	}
320	else {
321	// print header:
322	fputs("# arb field transfer set; version 1.0\n", out);
323	fputc('\n', out);
324
325	// print global RuleSet data:
326	{
327	ConstStrArray clines;
328	GBT_split_string(clines, comment.c_str(), '\n');
329
330	for (int c = 0; clines[c]; ++c) {
331	fprintf(out, "desc:%s\n", clines[c]);
332	}
333	fputc('\n', out);
334	}
335	fprintf(out, "transferUndef:%i\n", int(transferUndefFields));
336
337	// print rules:
338	for (size_t r = 0; r<size(); ++r) {
339	const Rule& rule = get(r);
340	string cfg = rule.getConfig();
341	fprintf(out, "rule:%s\n", cfg.c_str());
342	}
343	fputc('\n', out);
344
345	fclose(out);
346	}
347	return error;
348	}
349
350	inline bool isCommentLine(const string& line) {
351	size_t leadingSpaces = line.find_first_not_of(" \t");
352	return line[leadingSpaces] == '#';
353	}
354	inline bool shallIgnore(const string& line) {
355	// decide whether to ignore a line loaded from .fts file.
356	return line.empty() \|\| isCommentLine(line);
357	}
358
359	ErrorOrRuleSetPtr RuleSet::loadFrom(const char *filename) {
360	ARB_ERROR error;
361	RuleSetPtr ruleset;
362
363	FILE *in = fopen(filename, "rt");
364	if (!in) {
365	error = GB_IO_error("loading", filename);
366	}
367	else {
368	ruleset = new RuleSet();
369	BufferedFileReader reader(filename, in);
370
371	string line;
372	while (!error && reader.getLine(line)) {
373	if (shallIgnore(line)) continue;
374
375	size_t pos = line.find(':');
376	if (pos == string::npos) {
377	error = GBS_global_string("expected ':' while parsing line '%s'", line.c_str());
378	}
379	else {
380	string tag = line.substr(0, pos);
381	string content = line.substr(pos+1);
382
383	if (tag == "rule") {
384	ErrorOrRulePtr rule = Rule::makeFromConfig(content.c_str());
385	if (rule.hasError()) {
386	error = GBS_global_string("while reading rule from '%s': %s",
387	content.c_str(),
388	rule.getError().deliver());
389	}
390	else {
391	ruleset->add(rule.getValue());
392	}
393	}
394	else if (tag == "desc") {
395	const string& existing = ruleset->getComment();
396	ruleset->setComment(existing.empty() ? content : existing+'\n'+content);
397	}
398	else if (tag == "transferUndef") {
399	ruleset->set_transferUndefFields(bool(atoi(content.c_str())));
400	}
401	else {
402	error = GBS_global_string("unknown tag '%s' while parsing line '%s'",
403	tag.c_str(),
404	line.c_str());
405	}
406	}
407	}
408
409	if (error) ruleset.setNull();
410	}
411
412	return ErrorOrRuleSetPtr(error, ruleset);
413	}
414
415	// --------------------------------
416	// configuration of rules
417
418	#define SOURCE "source"
419	#define ACI "aci"
420	#define TARGET "target"
421	#define SEP "sep"
422	#define TYPE "type"
423	#define LOSS "loss"
424
425	#define PERMITTED "permitted"
426
427	inline const char *type2str(GB_TYPES type) {
428	const char *str = NULp;
429	switch (type) {
430	case GB_STRING: str = "text"; break;
431	case GB_INT: str = "int"; break;
432	case GB_FLOAT: str = "float"; break;
433	case GB_BITS: str = "bits"; break;
434	case GB_NONE: str = "auto"; break;
435	default: break;
436	}
437	return str;
438	}
439	inline GB_TYPES str2type(const char *str) {
440	GB_TYPES type = GB_TYPE_MAX; // invalid
441	switch (str[0]) {
442	case 't': if (strcmp(str, "text") == 0) type = GB_STRING; break;
443	case 'i': if (strcmp(str, "int") == 0) type = GB_INT; break;
444	case 'f': if (strcmp(str, "float") == 0) type = GB_FLOAT; break;
445	case 'b': if (strcmp(str, "bits") == 0) type = GB_BITS; break;
446	case 'a': if (strcmp(str, "auto") == 0) type = GB_NONE; break;
447	}
448	return type;
449	}
450
451	void ReadRule::saveReadConfig(ConfigMapping& cfgmap) const {
452	cfgmap.set_entry(SOURCE, fields);
453	if (separator != NOSEP) cfgmap.set_entry(SEP, separator);
454	if (!aci.empty()) cfgmap.set_entry(ACI, aci);
455	}
456
457	void WriteRule::saveWriteConfig(ConfigMapping& cfgmap) const {
458	cfgmap.set_entry(TARGET, name);
459	if (forcesType()) {
460	cfgmap.set_entry(TYPE, type2str(getTargetType()));
461	}
462	}
463	string Rule::getConfig() const {
464	ConfigMapping cfgmap;
465
466	saveReadConfig(cfgmap);
467	saveWriteConfig(cfgmap);
468
469	if (precisionLossPermitted()) {
470	cfgmap.set_entry(LOSS, PERMITTED);
471	}
472
473	return cfgmap.config_string();
474	}
475
476	ErrorOrRulePtr Rule::makeFromConfig(const char *config) {
477	RulePtr rule;
478	ConfigMapping cfgmap;
479	GB_ERROR error = cfgmap.parseFrom(config);
480
481	if (!error) {
482	const char *source = cfgmap.get_entry(SOURCE);
483	const char *target = cfgmap.get_entry(TARGET);
484	const char *sep = cfgmap.get_entry(SEP);
485
486	if (!source) error = "missing " SOURCE " entry";
487	if (!target) error = "missing " TARGET " entry";
488
489	if (!sep) sep = NOSEP; // default to 'no separator'
490
491	if (!error) {
492	const char *aci = cfgmap.get_entry(ACI);
493	if (aci) {
494	rule = makeAciConverter(source, sep, aci, target);
495	}
496	else {
497	rule = makeSimple(source, sep, target);
498	}
499
500	const char *typeID = cfgmap.get_entry(TYPE);
501	if (typeID) {
502	GB_TYPES type = str2type(typeID);
503	if (type == GB_TYPE_MAX) { // = unknown type ID
504	error = GBS_global_string("invalid type id '%s'", typeID);
505	rule.setNull();
506	}
507	else {
508	xf_assert(GB_TYPE_readable_as_string(type));
509	rule->setTargetType(type);
510	}
511	}
512
513	if (!error) {
514	const char *loss = cfgmap.get_entry(LOSS);
515	if (loss && strcmp(loss, PERMITTED) == 0) {
516	rule->permitPrecisionLoss();
517	}
518	}
519	}
520	}
521
522	return ErrorOrRulePtr(error, rule);
523	}
524
525
526	// ------------------------
527	// describe rules
528	string ReadRule::describe() const {
529	if (aci.empty()) return fields;
530	return fields+"\|ACI";
531	}
532	string WriteRule::describe() const {
533	return name;
534	}
535	string Rule::getShortDescription() const {
536	return ReadRule::describe() + " -> " + WriteRule::describe();
537	}
538
539	// -----------------------------
540	// ItemClonedByRuleSet
541	string ItemClonedByRuleSet::lastReportedError;
542
543	GB_ERROR ItemClonedByRuleSet::overlayOrCloneSub(const char subName, GBDATA gb_sub) {
544	GBDATA *gb_existing = GB_entry(gb_clone, subName);
545	GB_ERROR error;
546	if (gb_existing) { // if target entry exists ..
547	error = GB_copy_overlay(gb_existing, gb_sub); // .. overwrite its content.
548	}
549	else { // otherwise ..
550	error = GB_incur_error_if(!GB_clone(gb_clone, gb_sub)); // .. clone source entry
551	}
552	return error;
553	}
554
555	GB_ERROR ItemClonedByRuleSet::cloneMissingSub(const char subName, GBDATA gb_sub) {
556	GBDATA *gb_existing = GB_entry(gb_clone, subName);
557	GB_ERROR error;
558	if (gb_existing) { // if target entry exists ..
559	error = NULp; // .. keep it
560	}
561	else { // otherwise ..
562	error = GB_incur_error_if(!GB_clone(gb_clone, gb_sub)); // .. clone source entry
563	}
564	return error;
565	}
566
567	GB_ERROR ItemClonedByRuleSet::copySubIfMissing(const char *subName) {
568	// copy sub-field (or -container) if it doesn't exist in target
569	GB_ERROR error = NULp;
570	GBDATA *gb_sub = GB_entry(gb_source, subName);
571	if (!gb_sub) {
572	error = GBS_global_string("no such entry '%s' (in source)", subName);
573	UNCOVERED();
574	}
575	else {
576	error = cloneMissingSub(subName, gb_sub); // sub = passed field or container
577	}
578	return error;
579	}
580
581	GB_ERROR ItemClonedByRuleSet::copyAlignments() {
582	GB_ERROR error = NULp;
583	for (GBDATA *gb_ali = GB_child(gb_source); gb_ali; gb_ali = GB_nextChild(gb_ali)) {
584	if (GB_is_container(gb_ali)) {
585	const char *aliname = GB_read_key_pntr(gb_ali);
586	if (ARB_strBeginsWith(aliname, "ali_")) {
587	GBDATA *gb_data = GB_entry(gb_ali, "data");
588	if (gb_data) {
589	bool dataIsSTRING = GB_read_type(gb_data) == GB_STRING;
590	xf_assert(dataIsSTRING);
591	if (dataIsSTRING) {
592	error = overlayOrCloneSub(aliname, gb_ali); // sub = whole alignment container
593	}
594	}
595	else {
596	error = GB_incur_error();
597	UNCOVERED();
598	}
599	}
600	}
601	}
602	return error;
603	}
604
605	const char *ItemClonedByRuleSet::get_id_field() const {
606	const char *field = NULp;
607	switch (itemtype) {
608	case CLONE_ITEM_SPECIES: field = "name"; break;
609	default: xf_assert(0); break;
610	}
611	return field;
612	}
613
614	ItemClonedByRuleSet::ItemClonedByRuleSet(GBDATA& gb_item, ClonableItemType itemtype_, RuleSetPtr ruleset, ItemCloneType type_, GBDATA gb_refItem, const AlignmentTransporter *aliTransporter) :
615	itemtype(itemtype_),
616	gb_source(gb_item),
617	type(type_)
618	{
619	/*! clone or update item using ruleset.
620	*
621	* @param gb_item the source item (will be set to NULp if type_ is REPLACE_ITEM_BY_CLONE).
622	* @param itemtype_ currently always CLONE_ITEM_SPECIES.
623	* @param ruleset ruleset used to transfer fields from source item to cloned item
624	* @param type_ type of clone (see ItemCloneType for details).
625	* @param gb_refItem CLONE_INTO_EXISTING: target species, REAL_CLONE: target item container, otherwise: NULp
626	* @param aliTransporter allows to overide how alignment gets copied (default: copy all alignment sub-containers)
627	*/
628
629	// @@@ method is far too long -> split
630
631	GB_ERROR error = NULp;
632	GB_transaction ta(gb_source);
633
634	#if defined(ASSERTION_USED)
635	checked4error = false;
636	userCallbackUsed = false;
637	#endif
638
639	if (type == CLONE_INTO_EXISTING) {
640	if (gb_refItem) {
641	gb_clone = gb_refItem; // use passed clone as target
642	}
643	else {
644	error = "no target species specified (logic error)";
645	UNCOVERED();
646	}
647	}
648	else {
649	GBDATA *gb_item_container;
650	{
651	GBDATA *gb_src_item_container = GB_get_father(gb_source);
652	if (type == REAL_CLONE) {
653	gb_item_container = gb_refItem;
654	if (!gb_item_container) {
655	error = "no target item container specified (logic error)";
656	}
657	else if (gb_item_container == gb_src_item_container) {
658	error = "source and target item containers need to differ (logic error)";
659	}
660	}
661	else {
662	xf_assert(!gb_refItem); // passed value is ignored (please pass NULp)
663	gb_item_container = gb_src_item_container;
664	}
665	}
666
667	if (!error) {
668	xf_assert(itemtype_ == CLONE_ITEM_SPECIES); // next command only works for species
669	gb_clone = GB_create_container(gb_item_container, "species"); // create separate species
670	if (!gb_clone) {
671	error = GB_await_error();
672	UNCOVERED();
673	}
674	}
675	}
676
677	if (!error) {
678	// apply ruleset:
679	error = ruleset->transferBy(gb_source, gb_clone);
680
681	// perform some standard transfers:
682	const char *IDFIELD = get_id_field();
683	if (!error) error = copySubIfMissing(IDFIELD); // transfer IDFIELD for any itemtype
684
685	switch (itemtype) {
686	case CLONE_ITEM_SPECIES:
687	if (!error) error = copySubIfMissing("acc");
688	if (!error) {
689	if (aliTransporter) { // use user callback if given
690	if (aliTransporter->shallCopyBefore()) {
691	error = copyAlignments();
692	}
693	if (!error) {
694	error = aliTransporter->transport(gb_source, gb_clone); // e.g. used to adapt alignment in mergetool
695	}
696	#if defined(ASSERTION_USED)
697	userCallbackUsed = true;
698	#endif
699	}
700	else {
701	error = copyAlignments();
702	}
703	}
704	break;
705	default: xf_assert(0); break;
706	}
707
708	if (!error && ruleset->shallTransferUndefFields()) {
709
710	StrSet defined;
711	// extract used fields:
712	{
713	StrArray in, out;
714	ruleset->extractUsedFields(in, out);
715	// @@@ do extraction only once (not for each item transfer)
716
717	StrArray2StrSet(in, defined);
718	StrArray2StrSet(out, defined);
719	}
720	{
721	// exclude parent containers:
722	StrSet parents;
723	for (StrSet::const_iterator field = defined.begin(); field != defined.end(); ++field) {
724	size_t slashpos = field->find_first_of('/');
725	if (slashpos != string::npos) { // fieldname contains a slash
726	string parentname = field->substr(0, slashpos); // name of top-level parent container inside species
727	parents.insert(parentname);
728	}
729	}
730	defined.insert(parents.begin(), parents.end());
731	}
732
733	// transfer rest of fields (i.e. those neighter used by ruleset nor as standard field):
734	for (GBDATA *gb_field = GB_child(gb_source); gb_field && !error; gb_field = GB_nextChild(gb_field)) {
735	const char *key = GB_read_key_pntr(gb_field);
736	bool keyUsed = defined.find(key) != defined.end(); // key was read or written by ruleset
737
738	if (!keyUsed) {
739	error = copySubIfMissing(key);
740	}
741	}
742	}
743
744	// @@@ do we need to preserve security etc of cloned species? (security of sub-fields is preserved; e.g. see r17967)
745
746	if (!error) {
747	xf_assert(correlated(aliTransporter, userCallbackUsed)); // custom transporter was not used (logic error?)
748
749	switch (type) {
750	case REPLACE_ITEM_BY_CLONE:
751	error = GB_delete(gb_source); // will be replaced by clone
752	if (!error) {
753	gb_item = NULp;
754	gb_source = NULp;
755	}
756	break;
757
758	case RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS: {
759	GBDATA *gb_id = GB_entry(gb_source, IDFIELD);
760	if (!gb_id) {
761	error = GBS_global_string("expected field '%s' not found", IDFIELD);
762	}
763	else {
764	const char *name = GB_read_char_pntr(gb_id);
765	xf_assert(name);
766	orgName = name; // store name (written back in dtor)
767
768	error = GB_write_string(gb_id, "fake"); // change name // @@@ use different name
769	}
770	break;
771	}
772	case CLONE_INTO_EXISTING:
773	case REAL_CLONE:
774	// nothing to do here
775	break;
776	}
777	}
778	}
779
780	error = ta.close(error);
781	if (error) {
782	errorCopy = error; // store copy of error in string
783	gb_clone = NULp;
784	}
785	}
786
787	ItemClonedByRuleSet::~ItemClonedByRuleSet() {
788	if (!has_error()) { // if error occurred during construction -> TA was aborted -> nothing to undo
789	if (type == RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS) {
790	GB_transaction ta(gb_source);
791	GB_ERROR error = NULp;
792
793	GBDATA *gb_id = GB_entry(gb_source, get_id_field());
794	xf_assert(gb_id);
795
796	if (gb_id) {
797	xf_assert(!orgName.empty());
798	error = GB_write_string(gb_id, orgName.c_str());
799	if (error) {
800	fprintf(stderr, "Failed to rename original item after temp. clone (Reason: %s)", error);
801	xf_assert(0); // should not happen
802	}
803	}
804
805	// delete temp clone:
806	if (!error) {
807	error = GB_delete(gb_clone);
808	if (error) {
809	fprintf(stderr, "Failed to delete temp. clone (Reason: %s)", error);
810	xf_assert(0); // should not happen
811	}
812	}
813	}
814	}
815	}
816
817	};
818
819	// --------------------------------------------------------------------------------
820
821	#ifdef UNIT_TESTS
822
823	#include <arb_diff.h>
824	#include <arb_file.h>
825	#include <arb_defs.h>
826
827	#ifndef TEST_UNIT_H
828	#include <test_unit.h>
829	#endif
830
831	void TEST_type2id() {
832	using namespace FieldTransfer;
833
834	for (GB_TYPES t = GB_NONE; t<=GB_TYPE_MAX; t = GB_TYPES(t+1)) {
835	const char *id = type2str(t);
836	if (id) {
837	TEST_ANNOTATE(id);
838	TEST_EXPECT_EQUAL(str2type(id), t);
839	}
840	}
841	}
842	void TEST_transportedData() {
843	using namespace FieldTransfer;
844
845	GB_ERROR error;
846	{
847	TransportedData noData = TransportedData::none();
848	TransportedData errorData = TransportedData::makeError("the error msg");
849
850	TEST_REJECT(noData.failed());
851	TEST_REJECT(noData.exists());
852
853	TEST_EXPECT(errorData.failed());
854	error = errorData.getError();
855	TEST_EXPECT_EQUAL(error, "the error msg");
856	}
857	TEST_EXPECT_EQUAL(error, "the error msg"); // error has to survive destruction of TransportedData
858
859	TransportedData greet("hello");
860	TransportedData num(4711);
861	TransportedData fnum(0.815f);
862
863	TEST_REJECT(greet.failed());
864	TEST_EXPECT(greet.exists());
865	TEST_EXPECT_EQUAL(greet.getString(), "hello");
866
867	TEST_REJECT(num.failed());
868	TEST_EXPECT(num.exists());
869	TEST_EXPECT_EQUAL(num.getInt(), 4711);
870
871	TEST_REJECT(fnum.failed());
872	TEST_EXPECT(fnum.exists());
873	TEST_EXPECT_SIMILAR(fnum.getFloat(), 0.815, 0.000001);
874	}
875
876	void TEST_xferset() {
877	FieldTransfer::RuleSet fts;
878
879	TEST_EXPECT_ZERO(fts.size());
880	TEST_EXPECT(fts.empty());
881	TEST_EXPECT_EQUAL(fts.size(), 0);
882
883	// --------------------
884	// add rules:
885	fts.add(new FieldTransfer::Rule(FieldTransfer::ReadRule("location", NOSEP), // add a simple rule (one source, no ACI)
886	FieldTransfer::WriteRule("geolocation")));
887	TEST_EXPECT(!fts.empty());
888	TEST_EXPECT_EQUAL(fts.size(), 1);
889
890	fts.add(FieldTransfer::Rule::permitPrecisionLoss(new FieldTransfer::Rule(FieldTransfer::ReadRule("isolation", NOSEP, "upper"), // add an ACI rule (one source)
891	FieldTransfer::WriteRule("isolation_source", GB_INT)))); // force int type
892	TEST_EXPECT_EQUAL(fts.size(), 2);
893
894	// @@@ add multisource rules (with and w/o ACI)!
895
896	// ---------------------
897	// query rules
898	for (size_t r = 0; r<fts.size(); ++r) {
899	TEST_ANNOTATE(GBS_global_string("r=%zu", r));
900
901	const FieldTransfer::Rule& rule = fts.get(r);
902	switch (r) {
903	// @@@ add tests for source field(s)
904
905	case 0: // simple rule
906
907	TEST_EXPECT_EQUAL(rule.targetField(), "geolocation");
908	TEST_REJECT(rule.forcesType());
909	TEST_REJECT(rule.precisionLossPermitted());
910	break;
911
912	case 1: // basic ACI rule
913
914	TEST_EXPECT_EQUAL(rule.targetField(), "isolation_source");
915	TEST_EXPECT(rule.forcesType()); // type is forced ..
916	TEST_EXPECT_EQUAL(rule.getTargetType(), GB_INT); // .. to int
917	TEST_EXPECT(rule.precisionLossPermitted());
918	break;
919
920	default:
921	xf_assert(0); // untested rule
922	break;
923	}
924	}
925	TEST_ANNOTATE(NULp);
926	TEST_EXPECT_EQUAL(fts.size(), 2);
927
928	// -------------------------------------------
929	// test rule replacement and removal
930	{
931	// order=01
932	string cfg0 = fts.get(0).getConfig();
933	string cfg1 = fts.get(1).getConfig();
934
935	TEST_EXPECT_DIFFERENT(cfg0, cfg1); // otherwise test below does not test replacement
936
937	{
938	// swap 2 rules
939	FieldTransfer::RulePtr tmp = fts.getPtr(0);
940	fts.replace(0, fts.getPtr(1));
941	fts.replace(1, tmp);
942	// order=10
943	}
944
945	string newcfg0 = fts.get(0).getConfig();
946	string newcfg1 = fts.get(1).getConfig();
947
948	TEST_EXPECT_EQUAL(newcfg0, cfg1);
949	TEST_EXPECT_EQUAL(newcfg1, cfg0);
950
951	{
952	int insertedAt;
953
954	insertedAt = fts.insertBefore(0, fts.getPtr(1)); // insert before first -> order = 010
955	TEST_EXPECT_EQUAL(fts.size(), 3);
956	TEST_EXPECT_EQUAL(insertedAt, 0);
957	TEST_EXPECT_EQUAL(fts.get(insertedAt).getConfig(), cfg0);
958
959	insertedAt = fts.insertBefore(2, fts.getPtr(1)); // insert before last -> order = 0110
960	TEST_EXPECT_EQUAL(fts.size(), 4);
961	TEST_EXPECT_EQUAL(insertedAt, 2);
962	TEST_EXPECT_EQUAL(fts.get(insertedAt).getConfig(), cfg1);
963
964	insertedAt = fts.insertBefore(7, fts.getPtr(1)); // insert before invalid position = append -> order = 01101
965	TEST_EXPECT_EQUAL(fts.size(), 5);
966	TEST_EXPECT_EQUAL(insertedAt, 4);
967	TEST_EXPECT_EQUAL(fts.get(insertedAt).getConfig(), cfg1);
968
969	// "undo" inserts
970	fts.erase(1); // -> order = 0101
971	fts.erase(3); // erase at end -> order = 010
972	fts.erase(0); // -> order = 10
973	}
974
975	fts.erase(0); // erase 1st rule -> order = 0
976	TEST_EXPECT_EQUAL(fts.size(), 1);
977	string finalcfg = fts.get(0).getConfig();
978	TEST_EXPECT_EQUAL(finalcfg, cfg0);
979	}
980	}
981
982	class FailingRule: public FieldTransfer::Rule {
983	string partOfFailReason;
984	public:
985	FailingRule(const Rule& failing, string part) : Rule(failing), partOfFailReason(part) {}
986	FailingRule(FieldTransfer::RulePtr failing, string part) : Rule(*failing), partOfFailReason(part) {}
987	const char *expectedPartOfFailure() const { return partOfFailReason.c_str(); }
988	};
989
990
991	struct XferEnv : virtual Noncopyable { // provides test environment for transfer tests
992	GB_shell shell;
993
994	const char *target_ascii;
995
996	GBDATA *gb_src;
997	GBDATA *gb_dest;
998
999	XferEnv() :
1000	target_ascii("TEST_fields_xferred.arb")
1001	{
1002	gb_src = GB_open("TEST_fields_ascii.arb", "r"); // ../../UNIT_TESTER/run/TEST_fields_ascii.arb
1003	gb_dest = GB_open(target_ascii, "wc");
1004	}
1005	~XferEnv() {
1006	TEST_EXPECT_ZERO_OR_SHOW_ERRNO(GB_unlink(target_ascii));
1007
1008	GB_close(gb_dest);
1009	GB_close(gb_src);
1010	}
1011
1012	void transferAllSpeciesBy(const FieldTransfer::RuleSet& ruleset) { // transfer all species according to Ruleset
1013	// @@@ transferAllSpeciesBy is quite similar to what has to happen in merge-tool
1014	GB_transaction tas(gb_src);
1015	GB_transaction tad(gb_dest);
1016
1017	for (GBDATA *gb_src_species = GBT_first_species(gb_src);
1018	gb_src_species;
1019	gb_src_species = GBT_next_species(gb_src_species))
1020	{
1021	const char *name = GBT_get_name(gb_src_species);
1022	TEST_REJECT_NULL(name);
1023
1024	GBDATA *gb_dest_species = GBT_find_or_create_species(gb_dest, name, false);
1025	TEST_REJECT_NULL(gb_dest_species);
1026
1027	// @@@ transferAllSpeciesBy could allow overwrites (keep existing fields; allow overwrite of fields):
1028	// -> try to use ItemClonedByRuleSet with CLONE_INTO_EXISTING instead of direct call to transferBy below!
1029
1030	TEST_EXPECT_ZERO(GB_read_flag(gb_dest_species)); // (was previously done by GBT_find_or_create_species)
1031	TEST_EXPECT_NO_ERROR(ruleset.transferBy(gb_src_species, gb_dest_species));
1032	}
1033	}
1034
1035	// ----------------------------------------------------------------------------
1036
1037	void copyAllSpecies() {
1038	GB_transaction tas(gb_src);
1039	GB_transaction tad(gb_dest);
1040
1041	GBDATA *gb_dest_species_data = GBT_get_species_data(gb_dest);
1042	TEST_REJECT_NULL(gb_dest_species_data);
1043
1044	for (GBDATA *gb_src_species = GBT_first_species(gb_src);
1045	gb_src_species;
1046	gb_src_species = GBT_next_species(gb_src_species))
1047	{
1048	const char *name = GBT_get_name(gb_src_species);
1049	TEST_REJECT_NULL(name);
1050
1051	GBDATA *gb_dest_exists = GBT_find_species(gb_dest, name);
1052	TEST_EXPECT_NULL(gb_dest_exists); // this method cannot handle overwrites
1053
1054	GBDATA *gb_dest_species = GB_create_container(gb_dest_species_data, "species");
1055	TEST_REJECT_NULL(gb_dest_species);
1056
1057	TEST_EXPECT_NO_ERROR(GB_copy_dropProtectMarksAndTempstate(gb_dest_species, gb_src_species));
1058	TEST_EXPECT_ZERO(GB_read_flag(gb_dest_species));
1059	}
1060	}
1061
1062	// ----------------------------------------------------------------------------
1063	// write transferred data to ascii db + compare with expected result:
1064	void save() {
1065	TEST_EXPECT_NO_ERROR(GB_save_as(gb_dest, target_ascii, "a"));
1066	}
1067	void saveAndCompare(const char *expected_ascii, bool allowAutoUpdate) {
1068	save();
1069	if (allowAutoUpdate) {
1070	// #define TEST_AUTO_UPDATE // uncomment to update expected result
1071	#if defined(TEST_AUTO_UPDATE)
1072	TEST_COPY_FILE(target_ascii, expected_ascii);
1073	#endif
1074	}
1075	TEST_EXPECT_TEXTFILE_DIFFLINES(target_ascii, expected_ascii, 0);
1076	}
1077	};
1078
1079	static const char *expRuleConfig[] = {
1080	"source='lat_lon';target='geolocation'",
1081	"source='seq_quality_slv';target='seq/slv_quality'",
1082	"source='homop_slv';target='slv_homop'",
1083
1084	"source='no1';target='notTransferred'",
1085
1086	"source='pubmed_id';target='str2int';type='int'",
1087	"source='pubmed_id';target='str2flt';type='float'",
1088	"source='stop';target='int2flt';type='float'",
1089	"source='stop';target='int2str';type='text'",
1090	"source='align_ident_slv';target='flt2str';type='text'",
1091	"loss='permitted';source='align_ident_slv';target='flt2int';type='int'",
1092
1093	"aci='\|lower\|contains(partial)\|isAbove(0)';source='description';target='describedAsPartial'",
1094
1095	"aci='\|fdiv(2.0)';source='align_bp_score_slv';target='halfBPscoreStr'",
1096	"aci='\|fdiv(2.0)';source='align_bp_score_slv';target='halfBPscore';type='int'",
1097	"aci='\|fdiv(2.0)';source='align_bp_score_slv';target='halfBPscoreFlt';type='float'",
1098
1099	"aci='\|fmult(3.5)';source='homop_slv';target='multiHomopStr'",
1100	"aci='\|fmult(3.5)';source='homop_slv';target='multiHomopInt';type='int'",
1101	"aci='\|fmult(3.5)';source='homop_slv';target='multiHomop';type='float'",
1102
1103	"sep='/';source='embl_class;embl_division';target='embl_class_division'",
1104	"sep='-';source='align_startpos_slv;align_stoppos_slv';target='align_range_slv'",
1105	"sep='\\'';source='no1;align_bp_score_slv;no2;rel_ltp;no3';target='missing'",
1106
1107	"sep=';';source='NO1;no2;no3';target='skipped'",
1108
1109	"aci='\|upper';sep=':';source='embl_class;embl_division';target='emblClassDivision'",
1110	"aci='\|\"<\";dd;\">\"';sep=';';source='no1;no2;no3';target='skipped2'",
1111	};
1112
1113	static const char *EXPECTED_ASCII = "TEST_fields_xferred_expected.arb"; // ../../UNIT_TESTER/run/TEST_fields_xferred_expected.arb
1114	static const char *EXPECTED_ASCII_CLONED = "TEST_fields_cloned_expected.arb"; // ../../UNIT_TESTER/run/TEST_fields_cloned_expected.arb
1115
1116	void TEST_xferBySet() {
1117	// tests data transfer between items using RuleSet\|s
1118	using namespace FieldTransfer;
1119	XferEnv env;
1120
1121	// --------------------------------------------------------------
1122	// create rules and transfer item data using RuleSet\|s:
1123
1124	typedef std::vector<FailingRule> FailingRuleCont;
1125
1126	RuleSet ruleset;
1127	FailingRuleCont failing; // failing rules should go here
1128
1129	#define FAILING_add(rule,msg) failing.push_back(FailingRule(rule, msg))
1130
1131	ruleset.add(Rule::makeSimple("lat_lon", NOSEP, "geolocation")); // STRING->STRING
1132	ruleset.add(Rule::makeSimple("seq_quality_slv", NOSEP, "seq/slv_quality")); // INT ->INT (generate hierarchical target key)
1133	ruleset.add(Rule::makeSimple("homop_slv", NOSEP, "slv_homop")); // FLOAT ->FLOAT
1134
1135	ruleset.add(Rule::makeSimple("no1", NOSEP, "notTransferred")); // missing fields are skipped
1136
1137	// force target types
1138	ruleset.add(Rule::forceTargetType(GB_INT, Rule::makeSimple("pubmed_id", NOSEP, "str2int"))); // STRING->INT
1139	ruleset.add(Rule::forceTargetType(GB_FLOAT, Rule::makeSimple("pubmed_id", NOSEP, "str2flt"))); // STRING->FLOAT
1140	ruleset.add(Rule::forceTargetType(GB_FLOAT, Rule::makeSimple("stop", NOSEP, "int2flt"))); // INT->FLOAT
1141	ruleset.add(Rule::forceTargetType(GB_STRING, Rule::makeSimple("stop", NOSEP, "int2str"))); // INT->STRING
1142	ruleset.add(Rule::forceTargetType(GB_STRING, Rule::makeSimple("align_ident_slv", NOSEP, "flt2str"))); // FLOAT->STRING
1143	FAILING_add(Rule::forceTargetType(GB_INT, Rule::makeSimple("align_ident_slv", NOSEP, "dummy")), "lossy float->int type conversion (9.484605e+01->9.500000e+01)"); // FLOAT->INT
1144	ruleset.add(Rule::permitPrecisionLoss(Rule::forceTargetType(GB_INT, Rule::makeSimple("align_ident_slv", NOSEP, "flt2int")))); // FLOAT->INT
1145	// @@@ test forcedTargetType(GB_BITS)
1146
1147	// @@@ test transfer with existing target keys (and mismatching i.e. not default types)
1148	// -> shall force type conversion (e.g. int->float, float->string, string->int)
1149
1150	ruleset.add(Rule::makeAciConverter("description", NOSEP, "\|lower\|contains(partial)\|isAbove(0)", "describedAsPartial")); // transports STRING through ACI to STRING
1151
1152	// INT \| ACI -> STRING\|INT\|FLOAT:
1153	ruleset.add(Rule::makeAciConverter("align_bp_score_slv", NOSEP, "\|fdiv(2.0)", "halfBPscoreStr")); // transports INT through ACI to STRING
1154	ruleset.add(Rule::forceTargetType(GB_INT, Rule::makeAciConverter("align_bp_score_slv", NOSEP, "\|fdiv(2.0)", "halfBPscore"))); // transports INT through ACI to INT // @@@ why does this not complain about conversion loss? (e.g. PurGergo 58.5 -> 58). examine!!!
1155	ruleset.add(Rule::forceTargetType(GB_FLOAT, Rule::makeAciConverter("align_bp_score_slv", NOSEP, "\|fdiv(2.0)", "halfBPscoreFlt"))); // transports INT through ACI to FLOAT
1156
1157	// FLOAT \| ACI -> STRING:
1158	ruleset.add(Rule::makeAciConverter("homop_slv", NOSEP, "\|fmult(3.5)", "multiHomopStr")); // transports FLOAT through ACI to STRING
1159	ruleset.add(Rule::forceTargetType(GB_INT, Rule::makeAciConverter("homop_slv", NOSEP, "\|fmult(3.5)", "multiHomopInt"))); // transports FLOAT through ACI to INT // @@@ conversion loss happens (for all species?) // @@@ examine!
1160	ruleset.add(Rule::forceTargetType(GB_FLOAT, Rule::makeAciConverter("homop_slv", NOSEP, "\|fmult(3.5)", "multiHomop"))); // transports FLOAT through ACI to FLOAT
1161
1162	// @@@ test ACIs containing the following chars: ['"\\ = ]
1163
1164	// test concatenating rules:
1165	ruleset.add(Rule::makeSimple("embl_class;embl_division", "/", "embl_class_division")); // concat 2 STRINGs
1166	ruleset.add(Rule::makeSimple("align_startpos_slv;align_stoppos_slv", "-", "align_range_slv")); // concat 2 INTs
1167	ruleset.add(Rule::makeSimple("no1;align_bp_score_slv;no2;rel_ltp;no3", "'", "missing")); // concat INT + STRING (plus 3 non-existing fields)
1168
1169	ruleset.add(Rule::makeSimple("NO1;no2;no3", ";", "skipped")); // concat 3 non-existing fields -> field 'skipped' is NOT written to result DB
1170
1171	// test concatenation + ACI:
1172	ruleset.add(Rule::makeAciConverter("embl_class;embl_division", ":", "\|upper", "emblClassDivision")); // concat 2 STRINGs
1173	ruleset.add(Rule::makeAciConverter("no1;no2;no3", ";", "\|\"<\";dd;\">\"", "skipped2")); // concat 3 non-existing fields and apply ACI -> field 'skipped2' is NOT written to result DB + ACI not applied
1174
1175	// ----------------------------------------------------------------------------
1176	// please do not change 'ruleset' below this point
1177	// ----------------------------------------------------------------------------
1178
1179	// test input/output field extraction
1180	{
1181	StrArray input;
1182	StrArray output;
1183
1184	ruleset.extractUsedFields(input, output);
1185
1186	char *inJoined = GBT_join_strings(input, ';');
1187	char *outJoined = GBT_join_strings(output, ';');
1188
1189	TEST_EXPECT_EQUAL(inJoined, "align_bp_score_slv;align_ident_slv;align_startpos_slv;align_stoppos_slv;description;embl_class;embl_division;homop_slv;lat_lon;no1;no2;no3;pubmed_id;rel_ltp;seq_quality_slv;stop");
1190	TEST_EXPECT_EQUAL(outJoined, "align_range_slv;describedAsPartial;emblClassDivision;embl_class_division;flt2int;flt2str;geolocation;halfBPscore;halfBPscoreFlt;halfBPscoreStr;int2flt;int2str;missing;multiHomop;multiHomopInt;multiHomopStr;notTransferred;seq/slv_quality;skipped;skipped2;slv_homop;str2flt;str2int");
1191
1192	free(outJoined);
1193	free(inJoined);
1194	}
1195
1196	// convert all rules in 'ruleset' into string and test versus expRuleConfig:
1197	const size_t cfgs = ARRAY_ELEMS(expRuleConfig);
1198	const size_t rulz = ruleset.size();
1199	{
1200	const size_t testableRepr = min(cfgs, rulz);
1201	for (size_t r = 0; r<testableRepr; ++r) {
1202	TEST_ANNOTATE(GBS_global_string("r=%zu", r));
1203	const Rule& rule = ruleset.get(r);
1204	string rep = rule.getConfig();
1205	TEST_EXPECT_EQUAL(expRuleConfig[r], rep.c_str());
1206	}
1207	}
1208
1209	TEST_EXPECT_EQUAL(cfgs, rulz);
1210
1211	// test no 2 rules have equal config:
1212	for (size_t r1 = 0; r1<rulz; ++r1) {
1213	for (size_t r2 = r1+1; r2<rulz; ++r2) {
1214	TEST_ANNOTATE(GBS_global_string("r1/r2=%zu/%zu", r1, r2));
1215	TEST_EXPECT_DIFFERENT(expRuleConfig[r1], expRuleConfig[r2]);
1216	}
1217	}
1218	TEST_ANNOTATE(NULp);
1219
1220	env.transferAllSpeciesBy(ruleset);
1221
1222	// -------------------------------------------
1223	// test missing source-/target-item:
1224	{
1225	GB_transaction tas(env.gb_src);
1226	GB_transaction tad(env.gb_dest);
1227
1228	GBDATA *gb_src_species = GBT_first_species(env.gb_src);
1229	TEST_REJECT_NULL(gb_src_species);
1230
1231	const char *name = GBT_get_name(gb_src_species);
1232	TEST_REJECT_NULL(name);
1233
1234	GBDATA *gb_dest_species = GBT_find_species(env.gb_dest, name); // already has been created by 'transferAllSpeciesBy' above
1235	TEST_REJECT_NULL(gb_dest_species);
1236
1237	TEST_EXPECT_ERROR_CONTAINS(ruleset.transferBy(NULp, gb_dest_species), "lacking item to readFrom");
1238	TEST_EXPECT_ERROR_CONTAINS(ruleset.transferBy(gb_src_species, NULp), "lacking item to writeTo");
1239	}
1240
1241	// ---------------------------------------------
1242	// test rules failing during transfer:
1243	FAILING_add(Rule::forceTargetType(GB_FLOAT, Rule::makeSimple("nuc_region", NOSEP, "str2flt")), "cannot convert '1..1494' to float"); // test conversion errors (e.g. non-numeric string -> int or float)
1244	FAILING_add(Rule::makeAciConverter("homop_slv", NOSEP, "\|fmult(3.5, ooo)", "dummy"), "Unknown command '3.5'");
1245	FAILING_add(Rule::makeSimple("stop", NOSEP, "xxxx"), "Invalid character '' in 'xx*xx'");
1246	FAILING_add(Rule::makeSimple("ali_16s", NOSEP, "whatever"), "cannot read as data ('ali_16s' is a container)");
1247
1248	for (FailingRuleCont::const_iterator failRule = failing.begin(); failRule != failing.end(); ++failRule) {
1249	const FailingRule& testableRule = *failRule;
1250	RuleSet separated;
1251	separated.add(new Rule(testableRule));
1252
1253	// apply rule:
1254	{
1255	GB_transaction tas(env.gb_src);
1256	GB_transaction tad(env.gb_dest);
1257
1258	GB_ERROR error = NULp;
1259
1260	for (GBDATA *gb_src_species = GBT_first_species(env.gb_src);
1261	gb_src_species && !error;
1262	gb_src_species = GBT_next_species(gb_src_species))
1263	{
1264	const char *name = GBT_get_name(gb_src_species);
1265	GBDATA *gb_dest_species = GBT_find_species(env.gb_dest, name); // already has been created by 'transferBy' above
1266
1267	error = separated.transferBy(gb_src_species, gb_dest_species);
1268	}
1269	tad.close(error); // aborts transaction (if error occurs, which is expected here)
1270	TEST_EXPECT_ERROR_CONTAINS(error, testableRule.expectedPartOfFailure());
1271	}
1272	}
1273
1274	// ----------------------------------------------------------------
1275	// test type of each field is same across all items of DB
1276	{
1277	GB_transaction tad(env.gb_dest);
1278	GBDATA *gb_fake_species_data = GB_create_container(env.gb_dest, "tmp"); // necessary because GBT_scan_db never scans DIRECT childs
1279
1280	typedef map<string,GB_TYPES> TypedField;
1281	TypedField seen;
1282
1283	GB_ERROR error = NULp;
1284	for (GBDATA *gb_dest_species = GBT_first_species(env.gb_dest);
1285	gb_dest_species && !error;
1286	gb_dest_species = GBT_next_species(gb_dest_species))
1287	{
1288	const char *name = GBT_get_name(gb_dest_species);
1289	TEST_ANNOTATE(GBS_global_string("name=%s", name));
1290
1291	GBDATA *gb_specCopy = GB_create_container(gb_fake_species_data, "tmp");
1292	error = GB_copy_dropProtectMarksAndTempstate(gb_specCopy, gb_dest_species);
1293
1294	if (error) break;
1295
1296	StrArray curr;
1297	GBT_scan_db(curr, gb_fake_species_data, NULp);
1298	TEST_REJECT_ZERO(curr.size()); // expect fields
1299
1300	for (int i = 0; curr[i]; ++i) {
1301	const char *scanned = curr[i]; // 1st char is type
1302	const char *field = scanned+1;
1303	GB_TYPES type = GB_TYPES(scanned[0]);
1304
1305	TypedField::iterator found = seen.find(field);
1306	if (found != seen.end()) {
1307	if (type != found->second) {
1308	TEST_ANNOTATE(field);
1309	TEST_EXPECT_EQUAL(type, found->second); // existing field has to have same type (in all species)
1310	}
1311	}
1312	else {
1313	fprintf(stderr, "field='%s' type='%i'\n", field, type);
1314	seen[field] = type; // insert new field
1315	}
1316	}
1317
1318	if (!error) error = GB_delete(gb_specCopy);
1319	}
1320	if (!error) error = GB_delete(gb_fake_species_data);
1321	TEST_EXPECT_NO_ERROR(error);
1322	}
1323
1324	// ----------------------------------------------------------------------------
1325	xf_assert(rulz == ruleset.size()); // please do not change 'ruleset' after 'rulz' has been set!
1326
1327	env.saveAndCompare(EXPECTED_ASCII, true);
1328	}
1329
1330	void TEST_LATE_ruleConfigsReadable() {
1331	// run this test later than TEST_xferBySet
1332
1333	using namespace FieldTransfer;
1334
1335	{
1336	// test failing Rule configs:
1337	struct InvalidConfig {
1338	const char *config;
1339	GB_ERROR failure;
1340	};
1341	InvalidConfig invalidCfg[] = {
1342	{ TARGET "='xxx'", "missing source entry" },
1343	{ SOURCE "='xxx'", "missing target entry" },
1344	{ "tag='halfquot;", "could not find matching quote" },
1345	{ TARGET "='xxx';" SOURCE "='xxx';type='bizarre'", "invalid type id 'bizarre'" },
1346	};
1347
1348	for (size_t i = 0; i<ARRAY_ELEMS(invalidCfg); ++i) {
1349	InvalidConfig& CFG = invalidCfg[i];
1350	TEST_ANNOTATE(GBS_global_string("invalidCfg='%s'", CFG.config));
1351
1352	ErrorOrRulePtr result = Rule::makeFromConfig(CFG.config);
1353	TEST_EXPECT(result.hasError());
1354	TEST_EXPECT_ERROR_CONTAINS(result.getError(), CFG.failure);
1355	}
1356	TEST_ANNOTATE(NULp);
1357	}
1358
1359	const size_t cfgs = ARRAY_ELEMS(expRuleConfig);
1360	RuleSet ruleset;
1361
1362	// convert config->Rule + Rule->config + compare configs:
1363	for (size_t r = 0; r<cfgs; ++r) {
1364	const char *config = expRuleConfig[r];
1365
1366	ErrorOrRulePtr result = Rule::makeFromConfig(config);
1367	if (result.hasError()) {
1368	TEST_EXPECT_NO_ERROR(result.getError());
1369	}
1370	else {
1371	RulePtr rule = result.getValue();
1372	string reloadedConfig = rule->getConfig();
1373	TEST_EXPECT_EQUAL(reloadedConfig, config);
1374
1375	ruleset.add(rule);
1376	}
1377	}
1378
1379	// test RuleSet comment:
1380	const char *COMMENT = "A multi-\nline-\ntest-\ncomment.";
1381	ruleset.setComment(COMMENT);
1382	TEST_EXPECT_EQUAL(ruleset.getComment(), COMMENT);
1383
1384	ruleset.set_transferUndefFields(true);
1385	TEST_EXPECT(ruleset.shallTransferUndefFields());
1386
1387	// save RuleSet + reload it + compare:
1388	RuleSet reloaded_ruleset;
1389	{
1390	const char *rulesetSaved = "impexp/rulesetCurr.fts";
1391	const char *rulesetExpected = "impexp/ruleset.fts";
1392
1393	TEST_EXPECT_NO_ERROR(ruleset.saveTo(rulesetSaved));
1394	// #define TEST_AUTO_UPDATE_RS // uncomment to update expected result
1395	#if defined(TEST_AUTO_UPDATE_RS)
1396	TEST_COPY_FILE(rulesetSaved, rulesetExpected);
1397	#endif
1398	TEST_EXPECT_TEXTFILE_DIFFLINES(rulesetSaved, rulesetExpected, 0);
1399	TEST_EXPECT_ZERO_OR_SHOW_ERRNO(GB_unlink(rulesetSaved));
1400
1401	// reload RuleSet:
1402	{
1403	ErrorOrRuleSetPtr loaded = RuleSet::loadFrom(rulesetExpected);
1404	if (loaded.hasError()) TEST_EXPECT_NO_ERROR(loaded.getError()); // if error -> dump+fail
1405
1406	const RuleSet& loadedSet = *loaded.getValue();
1407	TEST_EXPECT_EQUAL(loadedSet.size(), ruleset.size());
1408
1409	// compare reloaded rules configs vs. array of expected configs.
1410	// tests:
1411	// - save+load is correct and complete
1412	// - Rule order is stable
1413	for (size_t r = 0; r<loadedSet.size(); ++r) {
1414	const Rule& rule = loadedSet.get(r);
1415	string cfg = rule.getConfig();
1416	TEST_EXPECT_EQUAL(cfg.c_str(), expRuleConfig[r]);
1417	}
1418
1419	// test comment survives reload:
1420	TEST_EXPECT_EQUAL(loadedSet.getComment(), COMMENT);
1421
1422	// test transferUndefFields survives save/load:
1423	TEST_EXPECT(loadedSet.shallTransferUndefFields());
1424
1425	// use reloaded Ruleset for tests below:
1426	reloaded_ruleset = loadedSet; // also tests RuleSet-copy-ctor works.
1427
1428	// test comment gets copied:
1429	TEST_EXPECT_EQUAL(reloaded_ruleset.getComment(), loadedSet.getComment());
1430	}
1431	}
1432
1433	// test RuleSet load/save errors:
1434	{
1435	const char *noSuchFile = "nosuch.fts";
1436	ErrorOrRuleSetPtr loaded = RuleSet::loadFrom(noSuchFile);
1437
1438	TEST_EXPECT(loaded.hasError());
1439	TEST_EXPECT_ERROR_CONTAINS(loaded.getError(), "No such file or directory");
1440
1441	const char *unsavable = "noSuchDir/whatever.fts";
1442	TEST_EXPECT_ERROR_CONTAINS(ruleset.saveTo(unsavable), "No such file or directory");
1443	}
1444
1445	// load empty file -> empty RuleSet
1446	{
1447	const char *emptyFile = "general/empty.input";
1448
1449	ErrorOrRuleSetPtr empty = RuleSet::loadFrom(emptyFile);
1450	TEST_REJECT(empty.hasError());
1451
1452	const RuleSet& emptySet = *empty.getValue();
1453	TEST_EXPECT_ZERO(emptySet.size()); // test emptySet has no rules
1454	TEST_EXPECT_EQUAL(emptySet.getComment(), ""); // test emptySet has no comment
1455	}
1456
1457	// use 'reloaded_ruleset' to modify same DB (as above in TEST_xferBySet):
1458	{
1459	XferEnv env;
1460	env.transferAllSpeciesBy(reloaded_ruleset);
1461	env.saveAndCompare(EXPECTED_ASCII, false); // if this fails -> saving/reloading config looses Rule information
1462	}
1463	}
1464
1465	#define CUSTOM_ALI_TRANSPORT_ERROR "custom ali transport error"
1466
1467	struct TestAlignmentTransporter FINAL_TYPE : public FieldTransfer::AlignmentTransporter {
1468	int mode;
1469	TestAlignmentTransporter(int mode_) : mode(mode_) {}
1470	bool shallCopyBefore() const OVERRIDE {
1471	return false; // do not call copyAlignments() b4 calling transport()
1472	}
1473	GB_ERROR transport(GBDATAgb_src_item, GBDATA gb_dst_item) const OVERRIDE {
1474	GB_ERROR error = NULp;
1475	switch (mode) {
1476	case 1: // custom error
1477	error = CUSTOM_ALI_TRANSPORT_ERROR;
1478	break;
1479
1480	case 2: // do nothing -> sequence still has old value (or is not copied)
1481	break;
1482
1483	case 3: { // write reverse sequence data
1484	GBDATA *gb_src_data = GBT_find_sequence(gb_src_item, "ali_16s");
1485	GBDATA *gb_dst_data = GBT_find_sequence(gb_dst_item, "ali_16s");
1486
1487	if (!gb_dst_data) { // destination has no 'ali_16s' -> clone whole container
1488	GBDATA *gb_src_ali = GB_get_father(gb_src_data);
1489
1490	error = GB_incur_error_if(!GB_clone(gb_dst_item, gb_src_ali));
1491	if (!error) {
1492	gb_dst_data = GBT_find_sequence(gb_dst_item, "ali_16s");
1493	xf_assert(gb_dst_data);
1494	}
1495	}
1496
1497	if (!error) {
1498	const char *seq = GB_read_char_pntr(gb_src_data);
1499	char *rev = GBT_reverseNucSequence(seq, strlen(seq));
1500
1501	error = GB_write_string(gb_dst_data, rev);
1502	free(rev);
1503	}
1504	break;
1505	}
1506	default: xf_assert(0); break; // unsupported mode
1507	}
1508	return error;
1509	}
1510	};
1511
1512	void TEST_clone_by_ruleset() {
1513	using namespace FieldTransfer;
1514
1515	RuleSetPtr ruleset;
1516	{
1517	const char *rulesetExpected = "impexp/ruleset.fts"; // same ruleset as used in tests above
1518
1519	ErrorOrRuleSetPtr loaded = RuleSet::loadFrom(rulesetExpected);
1520	if (loaded.hasError()) TEST_EXPECT_NO_ERROR(loaded.getError()); // if RuleSet load error -> dump+fail. see .@loadFrom
1521
1522	ruleset = loaded.getValue();
1523	}
1524
1525	// use 'ruleset' to modify same DB (but use ItemClonedByRuleSet here)
1526	{
1527	XferEnv env;
1528	env.copyAllSpecies(); // copy species of input DB -> output DB
1529
1530	GBDATA *gb_overwritten_species = NULp;
1531	char *overwrittenName = NULp;
1532
1533	// clone some species (inside output db):
1534	{
1535	GB_transaction ta(env.gb_dest);
1536
1537	GBDATA *gb_next_species = NULp;
1538	GBDATA *gb_first_clone = NULp;
1539	int count = 0;
1540
1541	for (GBDATA *gb_species = GBT_first_species(env.gb_dest);
1542	gb_species && gb_species != gb_first_clone;
1543	gb_species = gb_next_species, ++count)
1544	{
1545	gb_next_species = GBT_next_species(gb_species);
1546
1547	TEST_EXPECT_EQUAL(GB_countEntries(gb_species, "name"), 1); // safety-belt (had problems with duplicate name-entries)
1548
1549	char *orgName = ARB_strdup(GBT_get_name(gb_species));
1550	TEST_REJECT_NULL(orgName);
1551
1552	GBDATA *gb_clone = NULp;
1553	ItemCloneType cloneHow = (count == 3 \|\| count == 7) ? RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS : REPLACE_ITEM_BY_CLONE;
1554
1555	ruleset->set_transferUndefFields(count == 4); // test transfer of undefined fields for species #4
1556
1557	{
1558	ItemClonedByRuleSet clone(gb_species, CLONE_ITEM_SPECIES, ruleset, cloneHow, NULp, NULp);
1559
1560	if (clone.has_error()) TEST_EXPECT_NO_ERROR(clone.get_error());
1561	gb_clone = clone.get_clone();
1562	if (!gb_first_clone) {
1563	xf_assert(cloneHow == REPLACE_ITEM_BY_CLONE); // limit will not work otherwise
1564	gb_first_clone = gb_clone;
1565	}
1566
1567	switch (cloneHow) {
1568	case REPLACE_ITEM_BY_CLONE:
1569	TEST_EXPECT_NULL(gb_species);
1570	break;
1571	case RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS:
1572	TEST_EXPECT_EQUAL(GB_countEntries(gb_species, "name"), 1);
1573	TEST_EXPECT_EQUAL(GBT_get_name(gb_species), "fake"); // @@@ need a temporary name which cannot clash with existing names
1574	break;
1575	default:
1576	xf_assert(0); // not tested here
1577	break;
1578	}
1579
1580	TEST_EXPECT_EQUAL(GB_countEntries(gb_clone, "name"), 1);
1581	TEST_EXPECT_EQUAL(GBT_get_name(gb_clone), orgName);
1582	}
1583	// 'clone' has been destroyed now!
1584
1585	if (cloneHow == RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS) {
1586	TEST_EXPECT_EQUAL(GBT_get_name(gb_species), orgName); // rename back worked (RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS)
1587	}
1588	int orgNameCount = 0;
1589	for (GBDATA *gb_peek = GBT_first_species(env.gb_dest); gb_peek; gb_peek = GBT_next_species(gb_peek)) {
1590	bool hasOrgName = strcmp(GBT_get_name(gb_peek), orgName) == 0;
1591	orgNameCount += hasOrgName;
1592	switch (cloneHow) {
1593	case REPLACE_ITEM_BY_CLONE:
1594	if (hasOrgName) TEST_EXPECT(gb_peek == gb_clone); // orgName only used in persisting clone
1595	TEST_EXPECT(gb_peek != gb_species); // species has been removed
1596	break;
1597	case RENAME_ITEM_WHILE_TEMP_CLONE_EXISTS:
1598	if (hasOrgName) TEST_EXPECT(gb_peek == gb_species); // orgName only used in original species (i.e. temp. clone did vanish)
1599	TEST_EXPECT(gb_peek != gb_clone); // clone has been removed
1600	break;
1601	default:
1602	xf_assert(0); // not tested here
1603	break;
1604	}
1605	// @@@ also test against "fake" names?
1606	}
1607	TEST_EXPECT_EQUAL(orgNameCount, 1); // species with duplicate names unwanted
1608
1609	if (count == 3) {
1610	gb_overwritten_species = gb_species; // = copy of original
1611	overwrittenName = ARB_strdup(orgName);
1612	}
1613	free(orgName);
1614	}
1615	}
1616
1617	// test merging one species from source DB onto existing (cloned) species in dest DB:
1618	{
1619	GBDATA *gb_source_species;
1620
1621	{
1622	GB_transaction ta1(env.gb_src);
1623	GB_transaction ta2(env.gb_dest);
1624
1625	gb_source_species = GBT_find_species(env.gb_src, overwrittenName);
1626	TEST_REJECT_NULL(gb_source_species); // (in gb_src)
1627	TEST_REJECT_NULL(gb_overwritten_species); // (in gb_dest)
1628
1629	{
1630	GBDATA *gb_name = GB_entry(gb_overwritten_species, "name");
1631	TEST_EXPECT_NO_ERROR(GB_write_string(gb_name, "notOverwritten")); // prepare to test overwrite by data.
1632
1633	// modify "data" in "ali_16s" to prove overwrite later:
1634	GBDATA *gb_seq = GBT_find_sequence(gb_overwritten_species, "ali_16s");
1635	TEST_REJECT_NULL(gb_seq);
1636
1637	const char *seq = GB_read_char_pntr(gb_seq);
1638	char *seqMod = GBS_string_eval(seq, ":U=T");
1639
1640	TEST_EXPECT_NO_ERROR(GB_write_string(gb_seq, seqMod));
1641	free(seqMod);
1642	}
1643	}
1644
1645	SmartPtr<TestAlignmentTransporter> reverseAliTransporter;
1646
1647	// overwrite with result of ruleset (=> mix original and clone)
1648	for (int pass = 1; pass<=4; ++pass) {
1649	TEST_ANNOTATE(GBS_global_string("pass %i", pass));
1650
1651	GB_transaction ta1(env.gb_src);
1652	GB_transaction ta2(env.gb_dest);
1653
1654	SmartPtr<TestAlignmentTransporter> aliTransporter;
1655	if (pass<4) {
1656	aliTransporter = new TestAlignmentTransporter(pass);
1657	if (pass == 3) reverseAliTransporter = aliTransporter; // keep for later
1658	}
1659
1660	ItemClonedByRuleSet overclone(gb_source_species, CLONE_ITEM_SPECIES, ruleset, CLONE_INTO_EXISTING, gb_overwritten_species, aliTransporter.content());
1661
1662	if (pass == 1) {
1663	TEST_EXPECT(overclone.has_error());
1664	TEST_EXPECT_ERROR_CONTAINS(overclone.get_error(), CUSTOM_ALI_TRANSPORT_ERROR);
1665	ta2.close(overclone.get_error());
1666	ta1.close(overclone.get_error());
1667	}
1668	else {
1669	if (overclone.has_error()) TEST_EXPECT_NO_ERROR(overclone.get_error()); // expect no error, but show message if expectation fails
1670	TEST_EXPECT(overclone.get_clone() == gb_overwritten_species); // test get_clone reports gb_overwritten_species here
1671	TEST_EXPECT_EQUAL(GBT_get_name(gb_overwritten_species), "notOverwritten"); // test name of clone does not get overwritten
1672	TEST_EXPECT_EQUAL(GB_countEntries(gb_overwritten_species, "name"), 1);
1673
1674	{
1675	GBDATA *gb_seq = GBT_find_sequence(gb_overwritten_species, "ali_16s");
1676	TEST_REJECT_NULL(gb_seq);
1677
1678	const char *seq = GB_read_char_pntr(gb_seq);
1679
1680	switch (pass) {
1681	case 2: TEST_EXPECT_CONTAINS(seq, "GAAGTAGCTTGCTACTTTGCCGGCGAGCGGCGGAC"); break; // custom transporter: do nothing
1682	case 3: TEST_EXPECT_CONTAINS(seq, "CAGGCGGCGAGCGGCCGUUUCAUCGUUCGAUGAAG"); break; // custom transporter: writes reversed sequence data
1683	case 4: TEST_EXPECT_CONTAINS(seq, "GAAGUAGCUUGCUACUUUGCCGGCGAGCGGCGGAC"); break; // default behavior (=copy sequence over)
1684	default: xf_assert(0); break; // unexpected 'pass'
1685	}
1686
1687
1688	GBDATA *gb_ali = GB_get_father(gb_seq);
1689	TEST_EXPECT_EQUAL(GB_countEntries(gb_ali, "data"), 1);
1690	}
1691
1692	TEST_EXPECT_EQUAL(GB_countEntries(gb_overwritten_species, "ali_16s"), 1);
1693	}
1694	}
1695
1696	// "test" REAL_CLONE mode
1697
1698	{
1699	GB_transaction ta1(env.gb_src);
1700	GB_transaction ta2(env.gb_dest);
1701
1702	ItemClonedByRuleSet realClone(gb_source_species, CLONE_ITEM_SPECIES, ruleset, REAL_CLONE, GBT_get_species_data(env.gb_dest), &*reverseAliTransporter);
1703
1704	if (realClone.has_error()) TEST_EXPECT_NO_ERROR(realClone.get_error()); // expect no error, but show message if expectation fails
1705
1706	GBDATA *gb_clone = realClone.get_clone();
1707
1708	TEST_REJECT_NULL(gb_clone);
1709	TEST_REJECT_NULL(gb_source_species);
1710	TEST_EXPECT(gb_clone != gb_source_species);
1711	TEST_EXPECT_EQUAL(GBT_get_name(gb_clone), GBT_get_name(gb_source_species));
1712
1713	TEST_REJECT(GB_get_father(gb_clone) == GB_get_father(gb_source_species));
1714
1715	{
1716	GBDATA *gb_seq = GBT_find_sequence(gb_clone, "ali_16s");
1717	TEST_REJECT_NULL(gb_seq);
1718
1719	const char *seq = GB_read_char_pntr(gb_seq);
1720
1721	// TEST_EXPECT_CONTAINS(seq, "GAAGUAGCUUGCUACUUUGCCGGCGAGCGGCGGAC"); // default behavior (=copy sequence over)
1722	TEST_EXPECT_CONTAINS(seq, "CAGGCGGCGAGCGGCCGUUUCAUCGUUCGAUGAAG"); // custom transporter: writes reversed sequence data
1723
1724	GBDATA *gb_ali = GB_get_father(gb_seq);
1725	TEST_EXPECT_EQUAL(GB_countEntries(gb_ali, "data"), 1);
1726	}
1727	}
1728	}
1729
1730	env.saveAndCompare(EXPECTED_ASCII_CLONED, true);
1731	free(overwrittenName);
1732	}
1733	}
1734
1735	#endif // UNIT_TESTS
1736
1737	// --------------------------------------------------------------------------------

Note: See TracBrowser for help on using the repository browser.

Download in other formats: