]>
Commit | Line | Data |
---|---|---|
623bb90b | 1 | /************************************************************************** |
2 | * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. * | |
3 | * * | |
4 | * Author: The ALICE Off-line Project. * | |
5 | * Contributors are mentioned in the code where appropriate. * | |
6 | * * | |
7 | * Permission to use, copy, modify and distribute this software and its * | |
8 | * documentation strictly for non-commercial purposes is hereby granted * | |
9 | * without fee, provided that the above copyright notice appears in all * | |
10 | * copies and that both the copyright notice and this permission notice * | |
11 | * appear in the supporting documentation. The authors make no claims * | |
12 | * about the suitability of this software for any purpose. It is * | |
13 | * provided "as is" without express or implied warranty. * | |
14 | **************************************************************************/ | |
15 | ||
16 | /* $Id$ */ | |
17 | ||
18 | ////////////////////////////////////////////////////////////////////////// | |
19 | // marian.ivanov@cern.ch | |
20 | // Utilities for file merging. | |
21 | // Additional functionality on top of the standard TFileMerger: | |
22 | // | |
23 | // 1. Possibility to Set the reject/accept list. | |
24 | // 1.a) Only entries selected in accept list are merged. By default all entries are selected | |
25 | // use AddAccept 0 to specify your desired entry | |
26 | // 1.b) Entries selected in reject list are not merged. By default the reject list is empty. | |
27 | // | |
28 | // 2. syswatch.log is created diring mergin procedure. | |
29 | // Memeory consumption - for reading and for merging can be monitored | |
30 | ||
39f7cda2 | 31 | // RS: Changed merger to respect the structure of files being merged (directories, collections...) |
32 | // Additional option: SetNoTrees (default false) to not merge any tree | |
33 | // The code mostly taken from root's hadd.cxx | |
623bb90b | 34 | /* |
35 | Usage: | |
36 | // Libraries for all classes to be merged should be loaded before using the class | |
37 | gSystem->Load("libANALYSIS"); | |
38 | gSystem->Load("libANALYSIScalib"); | |
39 | gSystem->Load("libTPCcalib"); | |
40 | TH1::AddDirectory(0); | |
41 | ||
42 | //Example usage starting from the input data list in text file: | |
43 | // | |
44 | AliFileMerger merger; | |
45 | merger.AddReject("esdFriend"); | |
46 | merger.IterTXT("calib.list","CalibObjects.root",kFALSE); | |
47 | // | |
48 | ||
49 | */ | |
50 | ////////////////////////////////////////////////////////////////////////// | |
51 | ||
52 | ||
53 | #include <fstream> | |
39f7cda2 | 54 | #include <THashList.h> |
55 | #include <TChain.h> | |
56 | #include <TKey.h> | |
57 | #include <TH1.h> | |
58 | #include <THStack.h> | |
623bb90b | 59 | #include "TSystem.h" |
60 | #include "TFile.h" | |
61 | #include "TGrid.h" | |
62 | #include "TGridResult.h" | |
63 | #include "TObjString.h" | |
64 | #include "TObjArray.h" | |
65 | #include "TMethodCall.h" | |
39f7cda2 | 66 | #include "Riostream.h" |
623bb90b | 67 | #include "AliSysInfo.h" |
68 | #include "AliFileMerger.h" | |
3e486dd5 | 69 | #include "AliLog.h" |
623bb90b | 70 | |
71 | ClassImp(AliFileMerger) | |
72 | ||
3e486dd5 | 73 | ProcInfo_t procInfo;//TMP |
74 | ||
623bb90b | 75 | //////////////////////////////////////////////////////////////////////// |
76 | ||
77 | AliFileMerger::AliFileMerger(): | |
78 | TNamed(), | |
79 | fRejectMask(0), | |
39f7cda2 | 80 | fAcceptMask(0), |
3e486dd5 | 81 | fMaxFilesOpen(500), |
39f7cda2 | 82 | fNoTrees(kFALSE) |
623bb90b | 83 | { |
84 | // | |
85 | // Default constructor | |
86 | // | |
87 | } | |
88 | ||
89 | //______________________________________________________________________ | |
90 | ||
91 | AliFileMerger::AliFileMerger(const char* name): | |
92 | TNamed(name,name), | |
93 | fRejectMask(0), | |
39f7cda2 | 94 | fAcceptMask(0), |
3e486dd5 | 95 | fMaxFilesOpen(500), |
39f7cda2 | 96 | fNoTrees(kFALSE) |
623bb90b | 97 | { |
98 | // | |
99 | // | |
100 | // | |
101 | } | |
102 | ||
103 | ||
39f7cda2 | 104 | void AliFileMerger::IterAlien(const char* outputDir, const char* outputFileName, const char* pattern, Bool_t dontOverwrite){ |
623bb90b | 105 | |
106 | // | |
107 | // Merge the files coming out of the calibration job | |
108 | // | |
623bb90b | 109 | TString command; |
110 | // looking for files to be merged in the output directory | |
111 | command = Form("find %s/ *%s", outputDir, pattern); | |
112 | printf("command: %s\n", command.Data()); | |
113 | TGrid::Connect("alien://"); | |
114 | TGridResult *res = gGrid->Command(command); | |
115 | if (!res) return; | |
116 | TIter nextmap(res); | |
117 | TMap *map = 0; | |
118 | // loop over the results | |
39f7cda2 | 119 | TList sourcelist; |
120 | sourcelist.SetOwner(kTRUE); | |
121 | // | |
623bb90b | 122 | while((map=(TMap*)nextmap())) { |
123 | // getting the turl | |
124 | TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl")); | |
125 | if (!objs || !objs->GetString().Length()) { | |
126 | // Nothing found - skip this output | |
127 | delete res; | |
128 | break; | |
129 | } | |
130 | printf("looking for file %s\n",(objs->GetString()).Data()); | |
39f7cda2 | 131 | AddFile(&sourcelist, (objs->GetString()).Data());; |
623bb90b | 132 | } |
3e486dd5 | 133 | // |
134 | IterList(&sourcelist, outputFileName, dontOverwrite); | |
135 | delete res; | |
136 | } | |
137 | ||
138 | void AliFileMerger::IterList(const TList* namesList, const char* outputFileName, Bool_t dontOverwrite) | |
139 | { | |
140 | // merge in steps or in one go | |
141 | // | |
142 | gSystem->GetProcInfo(&procInfo); | |
143 | AliInfo(Form(">> memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
144 | // | |
145 | TString outputFile(outputFileName); | |
146 | gSystem->ExpandPathName(outputFile); | |
147 | // | |
148 | int nFiles = namesList->GetEntries(); | |
149 | int maxSrcOpen = fMaxFilesOpen - 1; | |
150 | TList filesList; | |
151 | filesList.SetOwner(kTRUE); | |
152 | // | |
153 | TString tmpDest[2] = {outputFile,outputFile}; // names for tmp files | |
154 | int npl = outputFile.Last('.'); | |
155 | if (npl<0) npl = outputFile.Length(); | |
156 | for (int i=0;i<2;i++) tmpDest[i].Insert(npl,Form("_TMPMERGE%d_",i)); | |
157 | // | |
158 | int nsteps = 0, currTmp = 0, start = 0; | |
159 | for (int ifl=0;ifl<nFiles;ifl++) { | |
160 | int st = ifl%maxSrcOpen; | |
161 | if (st==0 && ifl) { // new chunk should be started, merge what was already accumulated | |
162 | OpenNextChunks(namesList,&filesList,start,ifl-1); | |
163 | start = ifl; // remember where to start next step | |
164 | if (nsteps++) { // if not 1st one, merge the privous chunk with this one | |
165 | filesList.AddFirst(TFile::Open(tmpDest[currTmp].Data())); | |
166 | currTmp = (currTmp==0) ? 1:0; // swap tmp files | |
167 | } | |
168 | // open temp target | |
169 | TFile* targetTmp = TFile::Open( tmpDest[currTmp].Data(), "RECREATE"); | |
170 | if (!targetTmp || targetTmp->IsZombie()) { | |
171 | printf("Error opening temporary file %s\n",tmpDest[currTmp].Data()); | |
172 | return; | |
173 | } | |
174 | MergeRootfile(targetTmp, &filesList); | |
175 | targetTmp->Close(); | |
176 | delete targetTmp; | |
177 | filesList.Clear(); // close all open files | |
178 | } | |
179 | // nothing to do until needed amount of files is accumulated | |
180 | } | |
181 | // merge last step | |
39f7cda2 | 182 | TFile* target = TFile::Open( outputFile.Data(), (dontOverwrite ? "CREATE":"RECREATE") ); |
183 | if (!target || target->IsZombie()) { | |
184 | cerr << "Error opening target file (does " << outputFileName << " exist?)." << endl; | |
185 | cerr << "Use force = kTRUE to re-creation of output file." << endl; | |
186 | return; | |
3e486dd5 | 187 | } |
188 | OpenNextChunks(namesList,&filesList,start,nFiles-1); | |
189 | // add result of previous merges | |
190 | if (nsteps) filesList.AddFirst(TFile::Open(tmpDest[currTmp].Data())); | |
191 | MergeRootfile( target, &filesList); | |
192 | target->Close(); | |
39f7cda2 | 193 | delete target; |
3e486dd5 | 194 | filesList.Clear(); |
195 | // | |
196 | for (int i=0;i<2;i++) gSystem->Exec(Form("if [ -e %s ]; then \nrm %s\nfi",tmpDest[i].Data(),tmpDest[i].Data())); | |
197 | // | |
198 | printf("Merged %d files in %d steps\n",nFiles,++nsteps); | |
199 | // | |
200 | gSystem->GetProcInfo(&procInfo); | |
201 | AliInfo(Form("<< memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
623bb90b | 202 | } |
203 | ||
39f7cda2 | 204 | void AliFileMerger::IterTXT( const char * fileList, const char* outputFileName, Bool_t dontOverwrite){ |
623bb90b | 205 | |
206 | // Merge the files indicated in the list - fileList | |
207 | // ASCII file option example: | |
208 | // find `pwd`/ | grep AliESDfriends_v1.root > calib.list | |
209 | ||
623bb90b | 210 | // Open the input stream |
623bb90b | 211 | ifstream in; |
212 | in.open(fileList); | |
213 | // Read the input list of files | |
214 | TString objfile; | |
215 | Int_t counter=0; | |
39f7cda2 | 216 | TList sourcelist; |
217 | sourcelist.SetOwner(kTRUE); | |
623bb90b | 218 | while(in.good()) { |
219 | in >> objfile; | |
39f7cda2 | 220 | if (!objfile.Contains(".root")) continue; // protection |
221 | gSystem->ExpandPathName(objfile); | |
222 | printf("Add file:Counter\t%d\tMerging file %s\n",counter++,objfile.Data()); | |
223 | AddFile(&sourcelist, objfile.Data()); | |
623bb90b | 224 | } |
39f7cda2 | 225 | // |
3e486dd5 | 226 | IterList(&sourcelist, outputFileName, dontOverwrite); |
39f7cda2 | 227 | // |
623bb90b | 228 | } |
229 | ||
230 | void AliFileMerger::StoreResults(TObjArray * array, const char* outputFileName){ | |
231 | // | |
232 | // Storing the results in one single file | |
233 | // | |
234 | TFile *f = new TFile(outputFileName,"recreate"); | |
235 | for (Int_t i=0; i<array->GetEntries(); i++){ | |
236 | TObject *object0 = array->At(i); | |
237 | if (!object0) continue; | |
238 | object0->Write(); | |
239 | } | |
240 | f->Close(); | |
241 | delete f; | |
242 | } | |
243 | ||
244 | ||
245 | void AliFileMerger::StoreSeparateResults(TObjArray * array, const char* outputFileName){ | |
246 | // | |
247 | // Store the results in separate files (one per object) | |
248 | // | |
249 | for (Int_t i=0; i<array->GetEntries(); i++){ | |
250 | TObject *object0 = array->At(i); | |
251 | if (!object0) continue; | |
252 | TFile *f = new TFile(Form("%s_%s.root",outputFileName,object0->GetName()),"recreate"); | |
253 | object0->Write(); | |
254 | f->Close(); | |
255 | delete f; | |
256 | } | |
257 | } | |
258 | ||
623bb90b | 259 | void AliFileMerger::Merge(TFile* fileIn, TObjArray * array){ |
260 | // | |
261 | // Merging procedure | |
262 | // | |
96435124 | 263 | if (!array) return; |
623bb90b | 264 | static Int_t counter=-1; |
265 | counter++; | |
266 | TObjArray *carray = new TObjArray; //array of the objects inside current file | |
267 | carray->SetOwner(kTRUE); | |
268 | ||
269 | // load all objects to memory | |
270 | ||
271 | TList *farr = fileIn->GetListOfKeys(); | |
f8f4c15e | 272 | if (!farr) { |
273 | delete carray; | |
274 | return; | |
275 | } | |
623bb90b | 276 | for (Int_t ical=0; ical<farr->GetEntries(); ical++){ |
277 | if (!farr->At(ical)) continue; | |
278 | TString name(farr->At(ical)->GetName()); | |
279 | if (!IsAccepted(name)) continue; // skip not accepted entries | |
280 | TObject *obj = fileIn->Get(name.Data()); | |
281 | if (obj) carray->AddLast(obj); | |
282 | AliSysInfo::AddStamp(name.Data(),1,ical,counter); | |
283 | } | |
284 | ||
f8f4c15e | 285 | if (carray->GetEntries()==0) { |
286 | delete carray; | |
287 | return; | |
288 | } | |
623bb90b | 289 | TMethodCall callEnv; |
96435124 | 290 | Int_t entries =carray->GetEntriesFast(); |
291 | for (Int_t i=0; i<entries; i++){ | |
623bb90b | 292 | |
293 | TObjArray *templist = new TObjArray(1); | |
294 | templist->SetOwner(kFALSE); | |
295 | TObject *currentObject = carray->At(i); | |
f8f4c15e | 296 | if (!currentObject) { |
297 | delete templist; | |
298 | continue; | |
299 | } | |
623bb90b | 300 | printf("%s\n",currentObject->GetName()); |
301 | callEnv.InitWithPrototype(currentObject->IsA(), "Merge", "TCollection*"); | |
f8f4c15e | 302 | if (!callEnv.IsValid()) { |
303 | delete templist; | |
304 | continue; | |
305 | } | |
623bb90b | 306 | TString oname=currentObject->GetName(); |
307 | TObject *mergedObject = array->FindObject(currentObject->GetName()); | |
308 | if (!mergedObject) { | |
309 | array->AddLast(currentObject); | |
310 | carray->RemoveAt(i); | |
f8f4c15e | 311 | delete templist; |
623bb90b | 312 | continue; |
313 | } | |
314 | templist->AddLast(currentObject); | |
315 | callEnv.SetParam((Long_t) templist); | |
316 | callEnv.Execute(mergedObject); | |
317 | AliSysInfo::AddStamp(currentObject->GetName(),2,i,counter); | |
318 | delete templist; | |
319 | } | |
96435124 | 320 | carray->Delete(); |
623bb90b | 321 | delete carray; |
322 | } | |
323 | ||
324 | Bool_t AliFileMerger::IsAccepted(TString name){ | |
325 | // | |
326 | // Accept/reject logic | |
327 | // name - name of the entry | |
328 | // | |
329 | // if fAcceptMask specified - entry has to be in list of selected | |
330 | // if fRejectMask speciefied - entry with name speciief in the list are rejected | |
331 | // | |
332 | Bool_t accept=kTRUE; | |
333 | if (fAcceptMask){ | |
334 | // | |
335 | accept=kFALSE; | |
336 | for (Int_t iaccept=0; iaccept<fAcceptMask->GetEntries(); iaccept++){ | |
337 | if (name.Contains(fAcceptMask->At(iaccept)->GetName())) accept=kTRUE; // entry was selected | |
338 | } | |
339 | } | |
340 | if (!accept) return kFALSE; | |
341 | ||
342 | if (fRejectMask){ | |
343 | // | |
344 | for (Int_t ireject=0; ireject<fRejectMask->GetEntries(); ireject++){ | |
345 | if (name.Contains(fRejectMask->At(ireject)->GetName())) accept=kFALSE; // entry was rejected | |
346 | } | |
347 | } | |
348 | return accept; | |
349 | } | |
350 | ||
351 | ||
352 | ||
353 | ||
354 | void AliFileMerger::AddReject(const char *reject){ | |
355 | // | |
356 | // add reject string to the list of entries to be rejected for merging | |
357 | // | |
358 | if (!fRejectMask) fRejectMask = new TObjArray; | |
359 | fRejectMask->AddLast(new TObjString(reject)); | |
360 | } | |
361 | void AliFileMerger::AddAccept(const char *accept){ | |
362 | // | |
363 | // add reject string to the list of entries to be rejected for merging | |
364 | // | |
365 | if (!fAcceptMask) fAcceptMask = new TObjArray; | |
366 | fAcceptMask->AddLast(new TObjString(accept)); | |
367 | ||
368 | ||
369 | } | |
370 | ||
39f7cda2 | 371 | //___________________________________________________________________________ |
372 | int AliFileMerger::MergeRootfile( TDirectory *target, TList *sourcelist) | |
373 | { | |
374 | // Merge all objects in a directory | |
375 | // modified version of root's hadd.cxx | |
3e486dd5 | 376 | gSystem->GetProcInfo(&procInfo); |
377 | AliInfo(Form(">> memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
378 | ||
379 | ||
39f7cda2 | 380 | Int_t counterF = -1; |
381 | int status = 0; | |
382 | cout << "Target path: " << target->GetPath() << endl; | |
383 | TString path( (char*)strstr( target->GetPath(), ":" ) ); | |
384 | path.Remove( 0, 2 ); | |
385 | // | |
386 | // find 1st valid file | |
387 | TDirectory *first_source = (TDirectory*)sourcelist->First(); | |
388 | // | |
389 | Int_t nguess = sourcelist->GetSize()+1000; | |
390 | THashList allNames(nguess); | |
391 | ((THashList*)target->GetList())->Rehash(nguess); | |
392 | ((THashList*)target->GetListOfKeys())->Rehash(nguess); | |
393 | TList listH; | |
394 | TString listHargs; | |
395 | listHargs.Form("((TCollection*)0x%lx)", (ULong_t)&listH); | |
396 | // | |
397 | while(first_source) { | |
398 | counterF++; | |
399 | TDirectory *current_sourcedir = first_source->GetDirectory(path); | |
400 | if (!current_sourcedir) { | |
401 | first_source = (TDirectory*)sourcelist->After(first_source); | |
402 | continue; | |
403 | } | |
404 | // loop over all keys in this directory | |
405 | TChain *globChain = 0; | |
406 | TIter nextkey( current_sourcedir->GetListOfKeys() ); | |
407 | TKey *key, *oldkey=0; | |
408 | //gain time, do not add the objects in the list in memory | |
409 | TH1::AddDirectory(kFALSE); | |
410 | // | |
411 | int counterK = 0; | |
412 | // | |
413 | while ( (key = (TKey*)nextkey())) { | |
414 | if (current_sourcedir == target) break; | |
415 | // | |
416 | // check if we don't reject this name | |
417 | TString nameK(key->GetName()); | |
418 | if (!IsAccepted(nameK)) { | |
419 | if (!counterF) printf("Object %s is in rejection list, skipping...\n",nameK.Data()); | |
420 | continue; | |
421 | } | |
422 | // | |
423 | //keep only the highest cycle number for each key | |
424 | if (oldkey && !strcmp(oldkey->GetName(),key->GetName())) continue; | |
425 | if (!strcmp(key->GetClassName(),"TProcessID")) {key->ReadObj(); continue;} | |
426 | if (allNames.FindObject(key->GetName())) continue; | |
427 | TClass *cl = TClass::GetClass(key->GetClassName()); | |
428 | if (!cl || !cl->InheritsFrom(TObject::Class())) { | |
429 | cout << "Cannot merge object type, name: " | |
430 | << key->GetName() << " title: " << key->GetTitle() << endl; | |
431 | continue; | |
432 | } | |
433 | allNames.Add(new TObjString(key->GetName())); | |
434 | AliSysInfo::AddStamp(nameK.Data(),1,counterK++,counterF-1); | |
435 | // read object from first source file | |
436 | //current_sourcedir->cd(); | |
3e486dd5 | 437 | |
39f7cda2 | 438 | TObject *obj = key->ReadObj(); |
39f7cda2 | 439 | |
440 | if ( obj->IsA()->InheritsFrom( TTree::Class() ) ) { | |
441 | ||
442 | // loop over all source files create a chain of Trees "globChain" | |
443 | if (!fNoTrees) { // | |
444 | TString obj_name; | |
445 | if (path.Length()) { | |
446 | obj_name = path + "/" + obj->GetName(); | |
447 | } else { | |
448 | obj_name = obj->GetName(); | |
449 | } | |
450 | globChain = new TChain(obj_name); | |
451 | globChain->Add(first_source->GetName()); | |
452 | TFile *nextsource = (TFile*)sourcelist->After( first_source ); | |
453 | while ( nextsource ) { | |
454 | //do not add to the list a file that does not contain this Tree | |
455 | TFile *curf = TFile::Open(nextsource->GetName()); | |
456 | if (curf) { | |
457 | Bool_t mustAdd = kFALSE; | |
458 | if (curf->FindKey(obj_name)) { | |
459 | mustAdd = kTRUE; | |
460 | } else { | |
461 | //we could be more clever here. No need to import the object | |
462 | //we are missing a function in TDirectory | |
463 | TObject *aobj = curf->Get(obj_name); | |
464 | if (aobj) { mustAdd = kTRUE; delete aobj;} | |
465 | } | |
466 | if (mustAdd) { | |
467 | globChain->Add(nextsource->GetName()); | |
468 | } | |
469 | } | |
470 | delete curf; | |
471 | nextsource = (TFile*)sourcelist->After( nextsource ); | |
472 | } | |
473 | } | |
474 | } else if ( obj->IsA()->InheritsFrom( TDirectory::Class() ) ) { | |
475 | // it's a subdirectory | |
476 | ||
477 | cout << "Found subdirectory " << obj->GetName() << endl; | |
478 | // create a new subdir of same name and title in the target file | |
479 | target->cd(); | |
480 | TDirectory *newdir = target->mkdir( obj->GetName(), obj->GetTitle() ); | |
481 | ||
482 | // newdir is now the starting point of another round of merging | |
483 | // newdir still knows its depth within the target file via | |
484 | // GetPath(), so we can still figure out where we are in the recursion | |
485 | status = MergeRootfile( newdir, sourcelist); | |
486 | if (status) return status; | |
487 | ||
488 | } else if ( obj->InheritsFrom(TObject::Class()) | |
489 | && obj->IsA()->GetMethodWithPrototype("Merge", "TCollection*") ) { | |
490 | // object implements Merge(TCollection*) | |
491 | ||
492 | // loop over all source files and merge same-name object | |
493 | TFile *nextsource = (TFile*)sourcelist->After( first_source ); | |
494 | while ( nextsource ) { | |
495 | // make sure we are at the correct directory level by cd'ing to path | |
496 | TDirectory *ndir = nextsource->GetDirectory(path); | |
497 | if (ndir) { | |
498 | ndir->cd(); | |
499 | TKey *key2 = (TKey*)gDirectory->GetListOfKeys()->FindObject(key->GetName()); | |
500 | if (key2) { | |
501 | TObject *hobj = key2->ReadObj(); | |
502 | hobj->ResetBit(kMustCleanup); | |
503 | listH.Add(hobj); | |
504 | Int_t error = 0; | |
3e486dd5 | 505 | obj->Execute("Merge", listHargs.Data(), &error); // RS Probleme here |
39f7cda2 | 506 | if (error) { |
507 | cerr << "Error calling Merge() on " << obj->GetName() | |
508 | << " with the corresponding object in " << nextsource->GetName() << endl; | |
509 | } | |
510 | listH.Delete(); | |
511 | } | |
512 | } | |
513 | nextsource = (TFile*)sourcelist->After( nextsource ); | |
514 | } | |
515 | } else if ( obj->IsA()->InheritsFrom( THStack::Class() ) ) { | |
516 | THStack *hstack1 = (THStack*) obj; | |
517 | TList* l = new TList(); | |
518 | ||
519 | // loop over all source files and merge the histos of the | |
520 | // corresponding THStacks with the one pointed to by "hstack1" | |
521 | TFile *nextsource = (TFile*)sourcelist->After( first_source ); | |
522 | while ( nextsource ) { | |
523 | // make sure we are at the correct directory level by cd'ing to path | |
524 | TDirectory *ndir = nextsource->GetDirectory(path); | |
525 | if (ndir) { | |
526 | ndir->cd(); | |
527 | TKey *key2 = (TKey*)gDirectory->GetListOfKeys()->FindObject(hstack1->GetName()); | |
528 | if (key2) { | |
529 | THStack *hstack2 = (THStack*) key2->ReadObj(); | |
530 | l->Add(hstack2->GetHists()->Clone()); | |
531 | delete hstack2; | |
532 | } | |
533 | } | |
534 | ||
535 | nextsource = (TFile*)sourcelist->After( nextsource ); | |
536 | } | |
537 | hstack1->GetHists()->Merge(l); | |
538 | l->Delete(); | |
539 | } else { | |
540 | // object is of no type that we can merge | |
541 | cout << "Cannot merge object type, name: " | |
542 | << obj->GetName() << " title: " << obj->GetTitle() << endl; | |
543 | ||
544 | // loop over all source files and write similar objects directly to the output file | |
545 | TFile *nextsource = (TFile*)sourcelist->After( first_source ); | |
546 | while ( nextsource ) { | |
547 | // make sure we are at the correct directory level by cd'ing to path | |
548 | TDirectory *ndir = nextsource->GetDirectory(path); | |
549 | if (ndir) { | |
550 | ndir->cd(); | |
551 | TKey *key2 = (TKey*)gDirectory->GetListOfKeys()->FindObject(key->GetName()); | |
552 | if (key2) { | |
553 | TObject *nobj = key2->ReadObj(); | |
554 | nobj->ResetBit(kMustCleanup); | |
555 | int nbytes1 = target->WriteTObject(nobj, key2->GetName(), "SingleKey" ); | |
556 | if (nbytes1 <= 0) status = -1; | |
557 | delete nobj; | |
558 | } | |
559 | } | |
560 | nextsource = (TFile*)sourcelist->After( nextsource ); | |
561 | } | |
562 | } | |
563 | ||
564 | // now write the merged histogram (which is "in" obj) to the target file | |
565 | // note that this will just store obj in the current directory level, | |
566 | // which is not persistent until the complete directory itself is stored | |
567 | // by "target->Write()" below | |
568 | target->cd(); | |
569 | ||
570 | //!!if the object is a tree, it is stored in globChain... | |
571 | if(obj->IsA()->InheritsFrom( TDirectory::Class() )) { | |
572 | //printf("cas d'une directory\n"); | |
573 | } else if(obj->IsA()->InheritsFrom( TTree::Class() )) { | |
574 | if (!fNoTrees) { | |
575 | globChain->ls(); | |
576 | globChain->Merge(target->GetFile(),0,"keep fast"); | |
577 | delete globChain; | |
578 | } | |
579 | } else { | |
580 | int nbytes2 = obj->Write( key->GetName(), TObject::kSingleKey ); | |
581 | if (nbytes2 <= 0) status = -1; | |
582 | } | |
583 | oldkey = key; | |
584 | delete obj; | |
585 | } // while ( ( TKey *key = (TKey*)nextkey() ) ) | |
586 | first_source = (TDirectory*)sourcelist->After(first_source); | |
587 | } | |
588 | // save modifications to target file | |
589 | target->SaveSelf(kTRUE); | |
590 | // | |
3e486dd5 | 591 | gSystem->GetProcInfo(&procInfo); |
592 | AliInfo(Form("<< memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
593 | ||
39f7cda2 | 594 | return status; |
595 | } | |
596 | ||
597 | //___________________________________________________________________________ | |
3e486dd5 | 598 | int AliFileMerger::OpenNextChunks(const TList* namesList, TList* filesList, Int_t from, Int_t to) |
599 | { | |
600 | gSystem->GetProcInfo(&procInfo); | |
601 | AliInfo(Form(">> memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
602 | ||
603 | filesList->Clear(); | |
604 | int nEnt = namesList->GetEntries(); | |
605 | from = from<nEnt ? from : nEnt; | |
606 | to = to<nEnt ? to : nEnt; | |
607 | int count = 0; | |
608 | for (int i=from;i<=to;i++) { | |
609 | TNamed* fnam = (TNamed*)namesList->At(i); | |
610 | if (!fnam) continue; | |
611 | TString fnamS(fnam->GetName()); | |
612 | gSystem->ExpandPathName(fnamS); | |
613 | if (fnamS.BeginsWith("alien://") && !gGrid) TGrid::Connect("alien"); | |
614 | TFile* source = TFile::Open(fnam->GetName()); | |
615 | if( source==0 ) { printf("Failed to open file %s, will skip\n",fnam->GetName()); continue; } | |
616 | filesList->Add(source); | |
617 | printf("Opened file %s\n",fnam->GetName()); | |
618 | count++; | |
619 | } | |
620 | gSystem->GetProcInfo(&procInfo); | |
621 | AliInfo(Form("<< memory usage %ld %ld", procInfo.fMemResident, procInfo.fMemVirtual)); | |
622 | ||
623 | return count; | |
624 | } | |
625 | ||
626 | ||
627 | //___________________________________________________________________________ | |
628 | int AliFileMerger::AddFile(TList* namesList, std::string entry) | |
39f7cda2 | 629 | { |
630 | // add a new file to the list of files | |
631 | // static int count(0); | |
632 | if( entry.empty() ) return 0; | |
633 | size_t j =entry.find_first_not_of(' '); | |
634 | if( j==std::string::npos ) return 0; | |
635 | entry = entry.substr(j); | |
636 | if( entry.substr(0,1)=="@") { | |
637 | std::ifstream indirect_file(entry.substr(1).c_str() ); | |
638 | if( ! indirect_file.is_open() ) { | |
639 | std::cerr<< "Could not open indirect file " << entry.substr(1) << std::endl; | |
640 | return 1; | |
641 | } | |
642 | while( indirect_file ){ | |
643 | std::string line; | |
644 | std::getline(indirect_file, line); | |
3e486dd5 | 645 | if( AddFile(namesList, line)!=0 ) return 1;; |
39f7cda2 | 646 | } |
647 | return 0; | |
648 | } | |
649 | // cout << "Source file " << (++count) << ": " << entry << endl; | |
3e486dd5 | 650 | namesList->Add(new TNamed(entry,"")); |
39f7cda2 | 651 | return 0; |
652 | } |