]>
Commit | Line | Data |
---|---|---|
623bb90b | 1 | /************************************************************************** |
2 | * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. * | |
3 | * * | |
4 | * Author: The ALICE Off-line Project. * | |
5 | * Contributors are mentioned in the code where appropriate. * | |
6 | * * | |
7 | * Permission to use, copy, modify and distribute this software and its * | |
8 | * documentation strictly for non-commercial purposes is hereby granted * | |
9 | * without fee, provided that the above copyright notice appears in all * | |
10 | * copies and that both the copyright notice and this permission notice * | |
11 | * appear in the supporting documentation. The authors make no claims * | |
12 | * about the suitability of this software for any purpose. It is * | |
13 | * provided "as is" without express or implied warranty. * | |
14 | **************************************************************************/ | |
15 | ||
16 | /* $Id$ */ | |
17 | ||
18 | ////////////////////////////////////////////////////////////////////////// | |
19 | // marian.ivanov@cern.ch | |
20 | // Utilities for file merging. | |
21 | // Additional functionality on top of the standard TFileMerger: | |
22 | // | |
23 | // 1. Possibility to Set the reject/accept list. | |
24 | // 1.a) Only entries selected in accept list are merged. By default all entries are selected | |
25 | // use AddAccept 0 to specify your desired entry | |
26 | // 1.b) Entries selected in reject list are not merged. By default the reject list is empty. | |
27 | // | |
28 | // 2. syswatch.log is created diring mergin procedure. | |
29 | // Memeory consumption - for reading and for merging can be monitored | |
30 | ||
39f7cda2 | 31 | // RS: Changed merger to respect the structure of files being merged (directories, collections...) |
32 | // Additional option: SetNoTrees (default false) to not merge any tree | |
33 | // The code mostly taken from root's hadd.cxx | |
623bb90b | 34 | /* |
35 | Usage: | |
36 | // Libraries for all classes to be merged should be loaded before using the class | |
37 | gSystem->Load("libANALYSIS"); | |
38 | gSystem->Load("libANALYSIScalib"); | |
39 | gSystem->Load("libTPCcalib"); | |
40 | TH1::AddDirectory(0); | |
41 | ||
42 | //Example usage starting from the input data list in text file: | |
43 | // | |
44 | AliFileMerger merger; | |
45 | merger.AddReject("esdFriend"); | |
46 | merger.IterTXT("calib.list","CalibObjects.root",kFALSE); | |
47 | // | |
48 | ||
49 | */ | |
50 | ////////////////////////////////////////////////////////////////////////// | |
51 | ||
52 | ||
53 | #include <fstream> | |
39f7cda2 | 54 | #include <THashList.h> |
55 | #include <TChain.h> | |
56 | #include <TKey.h> | |
57 | #include <TH1.h> | |
58 | #include <THStack.h> | |
623bb90b | 59 | #include "TSystem.h" |
60 | #include "TFile.h" | |
61 | #include "TGrid.h" | |
62 | #include "TGridResult.h" | |
63 | #include "TObjString.h" | |
64 | #include "TObjArray.h" | |
65 | #include "TMethodCall.h" | |
39f7cda2 | 66 | #include "Riostream.h" |
623bb90b | 67 | #include "AliSysInfo.h" |
68 | #include "AliFileMerger.h" | |
3e486dd5 | 69 | #include "AliLog.h" |
623bb90b | 70 | |
c82bb898 | 71 | using std::cerr; |
72 | using std::endl; | |
73 | using std::cout; | |
74 | using std::ifstream; | |
623bb90b | 75 | ClassImp(AliFileMerger) |
76 | ||
3e486dd5 | 77 | ProcInfo_t procInfo;//TMP |
78 | ||
623bb90b | 79 | //////////////////////////////////////////////////////////////////////// |
80 | ||
81 | AliFileMerger::AliFileMerger(): | |
82 | TNamed(), | |
83 | fRejectMask(0), | |
39f7cda2 | 84 | fAcceptMask(0), |
bd7423bb | 85 | fMaxFilesOpen(800), |
39f7cda2 | 86 | fNoTrees(kFALSE) |
623bb90b | 87 | { |
88 | // | |
89 | // Default constructor | |
90 | // | |
91 | } | |
92 | ||
93 | //______________________________________________________________________ | |
94 | ||
95 | AliFileMerger::AliFileMerger(const char* name): | |
96 | TNamed(name,name), | |
97 | fRejectMask(0), | |
39f7cda2 | 98 | fAcceptMask(0), |
bd7423bb | 99 | fMaxFilesOpen(800), |
39f7cda2 | 100 | fNoTrees(kFALSE) |
623bb90b | 101 | { |
102 | // | |
103 | // | |
104 | // | |
105 | } | |
106 | ||
107 | ||
39f7cda2 | 108 | void AliFileMerger::IterAlien(const char* outputDir, const char* outputFileName, const char* pattern, Bool_t dontOverwrite){ |
623bb90b | 109 | |
110 | // | |
111 | // Merge the files coming out of the calibration job | |
112 | // | |
623bb90b | 113 | TString command; |
114 | // looking for files to be merged in the output directory | |
115 | command = Form("find %s/ *%s", outputDir, pattern); | |
116 | printf("command: %s\n", command.Data()); | |
117 | TGrid::Connect("alien://"); | |
118 | TGridResult *res = gGrid->Command(command); | |
119 | if (!res) return; | |
120 | TIter nextmap(res); | |
121 | TMap *map = 0; | |
122 | // loop over the results | |
39f7cda2 | 123 | TList sourcelist; |
124 | sourcelist.SetOwner(kTRUE); | |
125 | // | |
623bb90b | 126 | while((map=(TMap*)nextmap())) { |
127 | // getting the turl | |
128 | TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl")); | |
129 | if (!objs || !objs->GetString().Length()) { | |
130 | // Nothing found - skip this output | |
131 | delete res; | |
132 | break; | |
133 | } | |
134 | printf("looking for file %s\n",(objs->GetString()).Data()); | |
39f7cda2 | 135 | AddFile(&sourcelist, (objs->GetString()).Data());; |
623bb90b | 136 | } |
3e486dd5 | 137 | // |
138 | IterList(&sourcelist, outputFileName, dontOverwrite); | |
139 | delete res; | |
140 | } | |
141 | ||
142 | void AliFileMerger::IterList(const TList* namesList, const char* outputFileName, Bool_t dontOverwrite) | |
143 | { | |
144 | // merge in steps or in one go | |
145 | // | |
146 | gSystem->GetProcInfo(&procInfo); | |
147 | AliInfo(Form(">> memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
148 | // | |
149 | TString outputFile(outputFileName); | |
150 | gSystem->ExpandPathName(outputFile); | |
151 | // | |
152 | int nFiles = namesList->GetEntries(); | |
153 | int maxSrcOpen = fMaxFilesOpen - 1; | |
154 | TList filesList; | |
155 | filesList.SetOwner(kTRUE); | |
156 | // | |
157 | TString tmpDest[2] = {outputFile,outputFile}; // names for tmp files | |
158 | int npl = outputFile.Last('.'); | |
159 | if (npl<0) npl = outputFile.Length(); | |
160 | for (int i=0;i<2;i++) tmpDest[i].Insert(npl,Form("_TMPMERGE%d_",i)); | |
161 | // | |
162 | int nsteps = 0, currTmp = 0, start = 0; | |
163 | for (int ifl=0;ifl<nFiles;ifl++) { | |
164 | int st = ifl%maxSrcOpen; | |
165 | if (st==0 && ifl) { // new chunk should be started, merge what was already accumulated | |
166 | OpenNextChunks(namesList,&filesList,start,ifl-1); | |
167 | start = ifl; // remember where to start next step | |
168 | if (nsteps++) { // if not 1st one, merge the privous chunk with this one | |
169 | filesList.AddFirst(TFile::Open(tmpDest[currTmp].Data())); | |
170 | currTmp = (currTmp==0) ? 1:0; // swap tmp files | |
171 | } | |
172 | // open temp target | |
173 | TFile* targetTmp = TFile::Open( tmpDest[currTmp].Data(), "RECREATE"); | |
174 | if (!targetTmp || targetTmp->IsZombie()) { | |
175 | printf("Error opening temporary file %s\n",tmpDest[currTmp].Data()); | |
176 | return; | |
177 | } | |
178 | MergeRootfile(targetTmp, &filesList); | |
179 | targetTmp->Close(); | |
180 | delete targetTmp; | |
181 | filesList.Clear(); // close all open files | |
182 | } | |
183 | // nothing to do until needed amount of files is accumulated | |
184 | } | |
185 | // merge last step | |
39f7cda2 | 186 | TFile* target = TFile::Open( outputFile.Data(), (dontOverwrite ? "CREATE":"RECREATE") ); |
187 | if (!target || target->IsZombie()) { | |
188 | cerr << "Error opening target file (does " << outputFileName << " exist?)." << endl; | |
189 | cerr << "Use force = kTRUE to re-creation of output file." << endl; | |
190 | return; | |
3e486dd5 | 191 | } |
192 | OpenNextChunks(namesList,&filesList,start,nFiles-1); | |
193 | // add result of previous merges | |
194 | if (nsteps) filesList.AddFirst(TFile::Open(tmpDest[currTmp].Data())); | |
195 | MergeRootfile( target, &filesList); | |
196 | target->Close(); | |
39f7cda2 | 197 | delete target; |
3e486dd5 | 198 | filesList.Clear(); |
199 | // | |
200 | for (int i=0;i<2;i++) gSystem->Exec(Form("if [ -e %s ]; then \nrm %s\nfi",tmpDest[i].Data(),tmpDest[i].Data())); | |
201 | // | |
202 | printf("Merged %d files in %d steps\n",nFiles,++nsteps); | |
203 | // | |
204 | gSystem->GetProcInfo(&procInfo); | |
205 | AliInfo(Form("<< memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
623bb90b | 206 | } |
207 | ||
39f7cda2 | 208 | void AliFileMerger::IterTXT( const char * fileList, const char* outputFileName, Bool_t dontOverwrite){ |
623bb90b | 209 | |
210 | // Merge the files indicated in the list - fileList | |
211 | // ASCII file option example: | |
212 | // find `pwd`/ | grep AliESDfriends_v1.root > calib.list | |
213 | ||
623bb90b | 214 | // Open the input stream |
623bb90b | 215 | ifstream in; |
216 | in.open(fileList); | |
217 | // Read the input list of files | |
218 | TString objfile; | |
219 | Int_t counter=0; | |
39f7cda2 | 220 | TList sourcelist; |
221 | sourcelist.SetOwner(kTRUE); | |
623bb90b | 222 | while(in.good()) { |
223 | in >> objfile; | |
39f7cda2 | 224 | if (!objfile.Contains(".root")) continue; // protection |
225 | gSystem->ExpandPathName(objfile); | |
226 | printf("Add file:Counter\t%d\tMerging file %s\n",counter++,objfile.Data()); | |
227 | AddFile(&sourcelist, objfile.Data()); | |
623bb90b | 228 | } |
39f7cda2 | 229 | // |
3e486dd5 | 230 | IterList(&sourcelist, outputFileName, dontOverwrite); |
39f7cda2 | 231 | // |
623bb90b | 232 | } |
233 | ||
234 | void AliFileMerger::StoreResults(TObjArray * array, const char* outputFileName){ | |
235 | // | |
236 | // Storing the results in one single file | |
237 | // | |
238 | TFile *f = new TFile(outputFileName,"recreate"); | |
239 | for (Int_t i=0; i<array->GetEntries(); i++){ | |
240 | TObject *object0 = array->At(i); | |
241 | if (!object0) continue; | |
242 | object0->Write(); | |
243 | } | |
244 | f->Close(); | |
245 | delete f; | |
246 | } | |
247 | ||
248 | ||
249 | void AliFileMerger::StoreSeparateResults(TObjArray * array, const char* outputFileName){ | |
250 | // | |
251 | // Store the results in separate files (one per object) | |
252 | // | |
253 | for (Int_t i=0; i<array->GetEntries(); i++){ | |
254 | TObject *object0 = array->At(i); | |
255 | if (!object0) continue; | |
256 | TFile *f = new TFile(Form("%s_%s.root",outputFileName,object0->GetName()),"recreate"); | |
257 | object0->Write(); | |
258 | f->Close(); | |
259 | delete f; | |
260 | } | |
261 | } | |
262 | ||
623bb90b | 263 | void AliFileMerger::Merge(TFile* fileIn, TObjArray * array){ |
264 | // | |
265 | // Merging procedure | |
266 | // | |
96435124 | 267 | if (!array) return; |
623bb90b | 268 | static Int_t counter=-1; |
269 | counter++; | |
270 | TObjArray *carray = new TObjArray; //array of the objects inside current file | |
271 | carray->SetOwner(kTRUE); | |
272 | ||
273 | // load all objects to memory | |
274 | ||
275 | TList *farr = fileIn->GetListOfKeys(); | |
f8f4c15e | 276 | if (!farr) { |
277 | delete carray; | |
278 | return; | |
279 | } | |
623bb90b | 280 | for (Int_t ical=0; ical<farr->GetEntries(); ical++){ |
281 | if (!farr->At(ical)) continue; | |
282 | TString name(farr->At(ical)->GetName()); | |
283 | if (!IsAccepted(name)) continue; // skip not accepted entries | |
284 | TObject *obj = fileIn->Get(name.Data()); | |
285 | if (obj) carray->AddLast(obj); | |
286 | AliSysInfo::AddStamp(name.Data(),1,ical,counter); | |
287 | } | |
288 | ||
f8f4c15e | 289 | if (carray->GetEntries()==0) { |
290 | delete carray; | |
291 | return; | |
292 | } | |
623bb90b | 293 | TMethodCall callEnv; |
96435124 | 294 | Int_t entries =carray->GetEntriesFast(); |
295 | for (Int_t i=0; i<entries; i++){ | |
623bb90b | 296 | |
297 | TObjArray *templist = new TObjArray(1); | |
298 | templist->SetOwner(kFALSE); | |
299 | TObject *currentObject = carray->At(i); | |
f8f4c15e | 300 | if (!currentObject) { |
301 | delete templist; | |
302 | continue; | |
303 | } | |
623bb90b | 304 | printf("%s\n",currentObject->GetName()); |
305 | callEnv.InitWithPrototype(currentObject->IsA(), "Merge", "TCollection*"); | |
f8f4c15e | 306 | if (!callEnv.IsValid()) { |
307 | delete templist; | |
308 | continue; | |
309 | } | |
623bb90b | 310 | TString oname=currentObject->GetName(); |
311 | TObject *mergedObject = array->FindObject(currentObject->GetName()); | |
312 | if (!mergedObject) { | |
313 | array->AddLast(currentObject); | |
314 | carray->RemoveAt(i); | |
f8f4c15e | 315 | delete templist; |
623bb90b | 316 | continue; |
317 | } | |
318 | templist->AddLast(currentObject); | |
319 | callEnv.SetParam((Long_t) templist); | |
320 | callEnv.Execute(mergedObject); | |
321 | AliSysInfo::AddStamp(currentObject->GetName(),2,i,counter); | |
322 | delete templist; | |
323 | } | |
96435124 | 324 | carray->Delete(); |
623bb90b | 325 | delete carray; |
326 | } | |
327 | ||
328 | Bool_t AliFileMerger::IsAccepted(TString name){ | |
329 | // | |
330 | // Accept/reject logic | |
331 | // name - name of the entry | |
332 | // | |
333 | // if fAcceptMask specified - entry has to be in list of selected | |
334 | // if fRejectMask speciefied - entry with name speciief in the list are rejected | |
335 | // | |
336 | Bool_t accept=kTRUE; | |
337 | if (fAcceptMask){ | |
338 | // | |
339 | accept=kFALSE; | |
340 | for (Int_t iaccept=0; iaccept<fAcceptMask->GetEntries(); iaccept++){ | |
341 | if (name.Contains(fAcceptMask->At(iaccept)->GetName())) accept=kTRUE; // entry was selected | |
342 | } | |
343 | } | |
344 | if (!accept) return kFALSE; | |
345 | ||
346 | if (fRejectMask){ | |
347 | // | |
348 | for (Int_t ireject=0; ireject<fRejectMask->GetEntries(); ireject++){ | |
349 | if (name.Contains(fRejectMask->At(ireject)->GetName())) accept=kFALSE; // entry was rejected | |
350 | } | |
351 | } | |
352 | return accept; | |
353 | } | |
354 | ||
f059e38e | 355 | Bool_t AliFileMerger::IsRejected(TString name){ |
356 | // | |
357 | // check is the name is explicitly in the rejection list | |
358 | // if fRejectMask speciefied - entry with name speciief in the list are rejected | |
359 | // | |
360 | Bool_t reject=kFALSE; | |
361 | if (fRejectMask){ | |
362 | // | |
363 | for (Int_t ireject=0; ireject<fRejectMask->GetEntries(); ireject++){ | |
364 | if (name.Contains(fRejectMask->At(ireject)->GetName())) {reject=kTRUE; break;} // entry was rejected | |
365 | } | |
366 | } | |
367 | return reject; | |
368 | } | |
623bb90b | 369 | |
370 | ||
371 | ||
372 | void AliFileMerger::AddReject(const char *reject){ | |
373 | // | |
374 | // add reject string to the list of entries to be rejected for merging | |
375 | // | |
376 | if (!fRejectMask) fRejectMask = new TObjArray; | |
377 | fRejectMask->AddLast(new TObjString(reject)); | |
378 | } | |
379 | void AliFileMerger::AddAccept(const char *accept){ | |
380 | // | |
381 | // add reject string to the list of entries to be rejected for merging | |
382 | // | |
383 | if (!fAcceptMask) fAcceptMask = new TObjArray; | |
384 | fAcceptMask->AddLast(new TObjString(accept)); | |
385 | ||
386 | ||
387 | } | |
388 | ||
39f7cda2 | 389 | //___________________________________________________________________________ |
f796afad | 390 | int AliFileMerger::MergeRootfile( TDirectory *target, TList *sourcelist, Bool_t nameFiltering) |
39f7cda2 | 391 | { |
392 | // Merge all objects in a directory | |
393 | // modified version of root's hadd.cxx | |
3e486dd5 | 394 | gSystem->GetProcInfo(&procInfo); |
395 | AliInfo(Form(">> memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
22686bd6 | 396 | // |
39f7cda2 | 397 | int status = 0; |
398 | cout << "Target path: " << target->GetPath() << endl; | |
399 | TString path( (char*)strstr( target->GetPath(), ":" ) ); | |
400 | path.Remove( 0, 2 ); | |
401 | // | |
402 | // find 1st valid file | |
403 | TDirectory *first_source = (TDirectory*)sourcelist->First(); | |
404 | // | |
405 | Int_t nguess = sourcelist->GetSize()+1000; | |
406 | THashList allNames(nguess); | |
407 | ((THashList*)target->GetList())->Rehash(nguess); | |
408 | ((THashList*)target->GetListOfKeys())->Rehash(nguess); | |
409 | TList listH; | |
410 | TString listHargs; | |
411 | listHargs.Form("((TCollection*)0x%lx)", (ULong_t)&listH); | |
412 | // | |
413 | while(first_source) { | |
22686bd6 | 414 | // |
39f7cda2 | 415 | TDirectory *current_sourcedir = first_source->GetDirectory(path); |
416 | if (!current_sourcedir) { | |
417 | first_source = (TDirectory*)sourcelist->After(first_source); | |
418 | continue; | |
419 | } | |
420 | // loop over all keys in this directory | |
421 | TChain *globChain = 0; | |
422 | TIter nextkey( current_sourcedir->GetListOfKeys() ); | |
423 | TKey *key, *oldkey=0; | |
424 | //gain time, do not add the objects in the list in memory | |
425 | TH1::AddDirectory(kFALSE); | |
426 | // | |
427 | int counterK = 0; | |
22686bd6 | 428 | int counterF=0; |
39f7cda2 | 429 | // |
430 | while ( (key = (TKey*)nextkey())) { | |
431 | if (current_sourcedir == target) break; | |
432 | // | |
433 | // check if we don't reject this name | |
434 | TString nameK(key->GetName()); | |
f059e38e | 435 | if ((!IsAccepted(nameK) && nameFiltering) || (!nameFiltering && IsRejected(nameK))) { |
39f7cda2 | 436 | if (!counterF) printf("Object %s is in rejection list, skipping...\n",nameK.Data()); |
437 | continue; | |
438 | } | |
439 | // | |
440 | //keep only the highest cycle number for each key | |
441 | if (oldkey && !strcmp(oldkey->GetName(),key->GetName())) continue; | |
442 | if (!strcmp(key->GetClassName(),"TProcessID")) {key->ReadObj(); continue;} | |
443 | if (allNames.FindObject(key->GetName())) continue; | |
444 | TClass *cl = TClass::GetClass(key->GetClassName()); | |
445 | if (!cl || !cl->InheritsFrom(TObject::Class())) { | |
446 | cout << "Cannot merge object type, name: " | |
447 | << key->GetName() << " title: " << key->GetTitle() << endl; | |
448 | continue; | |
449 | } | |
69418387 | 450 | printf("Merging object %s, anchor directory: %s\n",key->GetName(),key->GetMotherDir()->GetPath()); |
39f7cda2 | 451 | allNames.Add(new TObjString(key->GetName())); |
22686bd6 | 452 | AliSysInfo::AddStamp(nameK.Data(),1,++counterK,counterF++); |
39f7cda2 | 453 | // read object from first source file |
454 | //current_sourcedir->cd(); | |
3e486dd5 | 455 | |
f059e38e | 456 | TDirectory* currDir = gDirectory; |
457 | key->GetMotherDir()->cd(); | |
39f7cda2 | 458 | TObject *obj = key->ReadObj(); |
f059e38e | 459 | currDir->cd(); |
c0339101 | 460 | if (!obj) { |
bd7423bb | 461 | AliError(Form("Failed to get the object with key %s from %s",key->GetName(),current_sourcedir->GetFile()->GetName())); |
c0339101 | 462 | continue; |
463 | } | |
464 | ||
39f7cda2 | 465 | if ( obj->IsA()->InheritsFrom( TTree::Class() ) ) { |
466 | ||
467 | // loop over all source files create a chain of Trees "globChain" | |
468 | if (!fNoTrees) { // | |
469 | TString obj_name; | |
470 | if (path.Length()) { | |
471 | obj_name = path + "/" + obj->GetName(); | |
472 | } else { | |
473 | obj_name = obj->GetName(); | |
474 | } | |
475 | globChain = new TChain(obj_name); | |
476 | globChain->Add(first_source->GetName()); | |
477 | TFile *nextsource = (TFile*)sourcelist->After( first_source ); | |
478 | while ( nextsource ) { | |
479 | //do not add to the list a file that does not contain this Tree | |
480 | TFile *curf = TFile::Open(nextsource->GetName()); | |
481 | if (curf) { | |
482 | Bool_t mustAdd = kFALSE; | |
483 | if (curf->FindKey(obj_name)) { | |
484 | mustAdd = kTRUE; | |
485 | } else { | |
486 | //we could be more clever here. No need to import the object | |
487 | //we are missing a function in TDirectory | |
488 | TObject *aobj = curf->Get(obj_name); | |
489 | if (aobj) { mustAdd = kTRUE; delete aobj;} | |
490 | } | |
491 | if (mustAdd) { | |
492 | globChain->Add(nextsource->GetName()); | |
493 | } | |
494 | } | |
495 | delete curf; | |
496 | nextsource = (TFile*)sourcelist->After( nextsource ); | |
497 | } | |
498 | } | |
499 | } else if ( obj->IsA()->InheritsFrom( TDirectory::Class() ) ) { | |
500 | // it's a subdirectory | |
501 | ||
502 | cout << "Found subdirectory " << obj->GetName() << endl; | |
503 | // create a new subdir of same name and title in the target file | |
504 | target->cd(); | |
505 | TDirectory *newdir = target->mkdir( obj->GetName(), obj->GetTitle() ); | |
506 | ||
507 | // newdir is now the starting point of another round of merging | |
508 | // newdir still knows its depth within the target file via | |
509 | // GetPath(), so we can still figure out where we are in the recursion | |
f796afad | 510 | status = MergeRootfile( newdir, sourcelist, kFALSE); |
39f7cda2 | 511 | if (status) return status; |
512 | ||
513 | } else if ( obj->InheritsFrom(TObject::Class()) | |
514 | && obj->IsA()->GetMethodWithPrototype("Merge", "TCollection*") ) { | |
515 | // object implements Merge(TCollection*) | |
516 | ||
517 | // loop over all source files and merge same-name object | |
518 | TFile *nextsource = (TFile*)sourcelist->After( first_source ); | |
519 | while ( nextsource ) { | |
520 | // make sure we are at the correct directory level by cd'ing to path | |
521 | TDirectory *ndir = nextsource->GetDirectory(path); | |
522 | if (ndir) { | |
523 | ndir->cd(); | |
524 | TKey *key2 = (TKey*)gDirectory->GetListOfKeys()->FindObject(key->GetName()); | |
525 | if (key2) { | |
526 | TObject *hobj = key2->ReadObj(); | |
c0339101 | 527 | if (!hobj) { |
528 | cout << "Failed to get the object with key " << key2->GetName() << " from " << | |
529 | ndir->GetFile()->GetName() << "/" << ndir->GetName() << endl; | |
bd7423bb | 530 | nextsource = (TFile*)sourcelist->After( nextsource ); |
c0339101 | 531 | continue; |
532 | } | |
533 | // | |
39f7cda2 | 534 | hobj->ResetBit(kMustCleanup); |
535 | listH.Add(hobj); | |
536 | Int_t error = 0; | |
3e486dd5 | 537 | obj->Execute("Merge", listHargs.Data(), &error); // RS Probleme here |
39f7cda2 | 538 | if (error) { |
539 | cerr << "Error calling Merge() on " << obj->GetName() | |
540 | << " with the corresponding object in " << nextsource->GetName() << endl; | |
541 | } | |
542 | listH.Delete(); | |
02098121 | 543 | // get the number of processed entries to be put in the syswatch.log |
544 | Double_t numberOfEntries = -1; | |
545 | if (obj->IsA()->GetMethodAllAny("GetEntries")) | |
546 | { | |
547 | TMethodCall getEntries(obj->IsA(), "GetEntries", ""); | |
548 | getEntries.Execute(obj, numberOfEntries); | |
549 | } | |
550 | AliSysInfo::AddStamp(nameK.Data(),1,counterK,counterF++,numberOfEntries); | |
39f7cda2 | 551 | } |
552 | } | |
553 | nextsource = (TFile*)sourcelist->After( nextsource ); | |
554 | } | |
555 | } else if ( obj->IsA()->InheritsFrom( THStack::Class() ) ) { | |
556 | THStack *hstack1 = (THStack*) obj; | |
557 | TList* l = new TList(); | |
558 | ||
559 | // loop over all source files and merge the histos of the | |
560 | // corresponding THStacks with the one pointed to by "hstack1" | |
561 | TFile *nextsource = (TFile*)sourcelist->After( first_source ); | |
562 | while ( nextsource ) { | |
563 | // make sure we are at the correct directory level by cd'ing to path | |
564 | TDirectory *ndir = nextsource->GetDirectory(path); | |
565 | if (ndir) { | |
566 | ndir->cd(); | |
567 | TKey *key2 = (TKey*)gDirectory->GetListOfKeys()->FindObject(hstack1->GetName()); | |
568 | if (key2) { | |
569 | THStack *hstack2 = (THStack*) key2->ReadObj(); | |
570 | l->Add(hstack2->GetHists()->Clone()); | |
571 | delete hstack2; | |
22686bd6 | 572 | AliSysInfo::AddStamp(nameK.Data(),1,counterK,counterF++); |
39f7cda2 | 573 | } |
574 | } | |
575 | ||
576 | nextsource = (TFile*)sourcelist->After( nextsource ); | |
577 | } | |
578 | hstack1->GetHists()->Merge(l); | |
579 | l->Delete(); | |
580 | } else { | |
581 | // object is of no type that we can merge | |
582 | cout << "Cannot merge object type, name: " | |
583 | << obj->GetName() << " title: " << obj->GetTitle() << endl; | |
584 | ||
585 | // loop over all source files and write similar objects directly to the output file | |
586 | TFile *nextsource = (TFile*)sourcelist->After( first_source ); | |
587 | while ( nextsource ) { | |
588 | // make sure we are at the correct directory level by cd'ing to path | |
589 | TDirectory *ndir = nextsource->GetDirectory(path); | |
590 | if (ndir) { | |
591 | ndir->cd(); | |
592 | TKey *key2 = (TKey*)gDirectory->GetListOfKeys()->FindObject(key->GetName()); | |
593 | if (key2) { | |
594 | TObject *nobj = key2->ReadObj(); | |
595 | nobj->ResetBit(kMustCleanup); | |
596 | int nbytes1 = target->WriteTObject(nobj, key2->GetName(), "SingleKey" ); | |
597 | if (nbytes1 <= 0) status = -1; | |
598 | delete nobj; | |
599 | } | |
600 | } | |
601 | nextsource = (TFile*)sourcelist->After( nextsource ); | |
602 | } | |
603 | } | |
604 | ||
605 | // now write the merged histogram (which is "in" obj) to the target file | |
606 | // note that this will just store obj in the current directory level, | |
607 | // which is not persistent until the complete directory itself is stored | |
608 | // by "target->Write()" below | |
609 | target->cd(); | |
610 | ||
611 | //!!if the object is a tree, it is stored in globChain... | |
612 | if(obj->IsA()->InheritsFrom( TDirectory::Class() )) { | |
613 | //printf("cas d'une directory\n"); | |
614 | } else if(obj->IsA()->InheritsFrom( TTree::Class() )) { | |
615 | if (!fNoTrees) { | |
616 | globChain->ls(); | |
617 | globChain->Merge(target->GetFile(),0,"keep fast"); | |
618 | delete globChain; | |
619 | } | |
620 | } else { | |
621 | int nbytes2 = obj->Write( key->GetName(), TObject::kSingleKey ); | |
622 | if (nbytes2 <= 0) status = -1; | |
623 | } | |
624 | oldkey = key; | |
625 | delete obj; | |
626 | } // while ( ( TKey *key = (TKey*)nextkey() ) ) | |
627 | first_source = (TDirectory*)sourcelist->After(first_source); | |
628 | } | |
629 | // save modifications to target file | |
630 | target->SaveSelf(kTRUE); | |
631 | // | |
3e486dd5 | 632 | gSystem->GetProcInfo(&procInfo); |
633 | AliInfo(Form("<< memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
634 | ||
39f7cda2 | 635 | return status; |
636 | } | |
637 | ||
638 | //___________________________________________________________________________ | |
3e486dd5 | 639 | int AliFileMerger::OpenNextChunks(const TList* namesList, TList* filesList, Int_t from, Int_t to) |
640 | { | |
641 | gSystem->GetProcInfo(&procInfo); | |
642 | AliInfo(Form(">> memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
643 | ||
644 | filesList->Clear(); | |
645 | int nEnt = namesList->GetEntries(); | |
646 | from = from<nEnt ? from : nEnt; | |
647 | to = to<nEnt ? to : nEnt; | |
648 | int count = 0; | |
649 | for (int i=from;i<=to;i++) { | |
650 | TNamed* fnam = (TNamed*)namesList->At(i); | |
651 | if (!fnam) continue; | |
652 | TString fnamS(fnam->GetName()); | |
653 | gSystem->ExpandPathName(fnamS); | |
654 | if (fnamS.BeginsWith("alien://") && !gGrid) TGrid::Connect("alien"); | |
655 | TFile* source = TFile::Open(fnam->GetName()); | |
656 | if( source==0 ) { printf("Failed to open file %s, will skip\n",fnam->GetName()); continue; } | |
657 | filesList->Add(source); | |
658 | printf("Opened file %s\n",fnam->GetName()); | |
659 | count++; | |
660 | } | |
661 | gSystem->GetProcInfo(&procInfo); | |
662 | AliInfo(Form("<< memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
663 | ||
664 | return count; | |
665 | } | |
666 | ||
667 | ||
668 | //___________________________________________________________________________ | |
669 | int AliFileMerger::AddFile(TList* namesList, std::string entry) | |
39f7cda2 | 670 | { |
671 | // add a new file to the list of files | |
672 | // static int count(0); | |
673 | if( entry.empty() ) return 0; | |
674 | size_t j =entry.find_first_not_of(' '); | |
675 | if( j==std::string::npos ) return 0; | |
676 | entry = entry.substr(j); | |
677 | if( entry.substr(0,1)=="@") { | |
678 | std::ifstream indirect_file(entry.substr(1).c_str() ); | |
679 | if( ! indirect_file.is_open() ) { | |
680 | std::cerr<< "Could not open indirect file " << entry.substr(1) << std::endl; | |
681 | return 1; | |
682 | } | |
683 | while( indirect_file ){ | |
684 | std::string line; | |
685 | std::getline(indirect_file, line); | |
3e486dd5 | 686 | if( AddFile(namesList, line)!=0 ) return 1;; |
39f7cda2 | 687 | } |
688 | return 0; | |
689 | } | |
690 | // cout << "Source file " << (++count) << ": " << entry << endl; | |
3e486dd5 | 691 | namesList->Add(new TNamed(entry,"")); |
39f7cda2 | 692 | return 0; |
693 | } |