1 /**************************************************************************
2 * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
19 //TOOLKIT for chain manipulation:
22 // 1. Check the list of files
23 AliXRDPROOFtoolkit toolkit;
24 AliXRDPROOFtoolkit::FilterList("pp.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",0)
25 AliXRDPROOFtoolkit::FilterList("pp.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",1)
27 //2. make a chain or random chain form the list of files
28 TChain * chain = toolkit.MakeChain("esd.txt","esdTree",0,10)
29 TChain * chainRandom = toolkit.MakeChainrandom("esd.txt","esdTree",0,10)
30 chain->Draw("fTPCnclsF");
37 #include <TObjArray.h>
38 #include <TObjString.h>
50 #include <TTimeStamp.h>
51 #include <AliXRDPROOFtoolkit.h>
54 ClassImp(AliXRDPROOFtoolkit)
58 //______________________________________________________________________________
59 AliXRDPROOFtoolkit::AliXRDPROOFtoolkit () :
61 fVerbose(kFALSE), // verbso mode - print command
62 fUserName(""), // user name
63 fUserGroup(0) // user group info
68 fUserGroup = gSystem->GetUserInfo();
69 fUserName = fUserGroup->fUser;
77 TChain* AliXRDPROOFtoolkit::MakeChain(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles, Int_t startFile)
82 TChain* chain = new TChain(treeName);
84 // Open the input stream
88 // Read the input list of files and add them to the chain
97 if (!currentFile.Contains("root")) continue; // protection
99 if (counter<startFile) continue;
100 if (counter>maxFiles+startFile) break;
101 TFile * f = TFile::Open(currentFile.Data());
103 chain->Add(currentFile.Data());
113 TChain* AliXRDPROOFtoolkit::MakeChainRandom(const char*fileIn, const char * treeName,const char *fName, Int_t maxFiles, Int_t startFile)
116 // Create a TDSet - files are in random order
118 // filein - input list text file
119 // treename - containg tree
120 // maxFiles - maximum number of files included
122 TObjArray array(10000);
124 TChain* chain = new TChain(treeName);
126 // Open the input stream
130 // Read the input list of files and add them to the chain
139 if (!currentFile.Contains("root")) continue; // protection
141 // chain->Add(currentFile.Data());
142 array.AddLast(new TObjString(currentFile));
145 Int_t entries = array.GetEntries();
146 printf("Number of entries\t%d\n",entries);
150 Double_t *randomI = new Double_t[entries];
151 Int_t *indexes = new Int_t[entries];
152 for (Int_t i=0;i<entries; i++) randomI[i]=gRandom->Rndm();
153 TMath::Sort(entries,randomI,indexes);
155 for (Int_t i=startFile; (i<startFile+maxFiles) && (i<entries); i++){
156 Int_t ifile = indexes[i];
157 if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
158 printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
159 chain->Add(array.At(ifile)->GetName());
160 array.At(ifile)->SetBit(TObject::kCannotPick);
168 TDSet* AliXRDPROOFtoolkit::MakeSet(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
171 // Create the TDSet out of list
172 // filein - input list text file
173 // treename - containg tree
174 // maxFiles - maximum number of files included
176 TDSet* chain = new TDSet(treeName);
178 // Open the input stream
182 // Read the input list of files and add them to the chain
191 if (!currentFile.Contains("root")) continue; // protection
193 if (maxFiles>0 && counter>maxFiles) break;
194 chain->Add(currentFile.Data());
203 TDSet* AliXRDPROOFtoolkit::MakeSetRandom(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
206 // Create a TDSet - files are in random order
208 // filein - input list text file
209 // treename - containg tree
210 // maxFiles - maximum number of files included
212 TObjArray array(10000);
214 TDSet* chain = new TDSet(treeName);
216 // Open the input stream
220 // Read the input list of files and add them to the chain
229 if (!currentFile.Contains("root")) continue; // protection
231 // chain->Add(currentFile.Data());
232 array.AddLast(new TObjString(currentFile));
235 Int_t entries = array.GetEntries();
236 printf("Number of entries\t%d",entries);
237 if (maxFiles<0) maxFiles=entries;
238 if (maxFiles>entries) maxFiles=entries;
239 for (Int_t i=0; i<maxFiles; i++){
240 Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
241 if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
242 printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
243 chain->Add(array.At(ifile)->GetName());
244 array.At(ifile)->SetBit(TObject::kCannotPick);
259 Int_t AliXRDPROOFtoolkit::CheckTreeInFile(const char*fileName,const char*treeName, Int_t debugLevel, const char *branchName){
261 // Check the tree in file
262 // fileName - the name of the file with tree
263 // treeName - the name of file
264 // debugLevel - 0 check the existance of the file - 1 make loop over entries
265 // branchName - if debugLevel>0 the branch is chcecked
266 // if brnachName =0 the content of full tree is chcecked
267 // return value = 0 - Check things OK
268 // -1 - file not exist or not accesible
269 // -2 - file is zombie
270 // -3 - tree not present
271 // -4 - branch not present
272 TFile * file = TFile::Open(fileName);
273 if (!file) { return -1;}
274 if (file->IsZombie()) {file->Close(); delete file; return -2;};
276 TString TrName(treeName);
278 //cout <<" treename ==== *"<<endl;;
279 file->Close(); delete file;
282 TTree * tree = (TTree*)file->Get(treeName);
283 if (!tree) {file->Close(); delete file; return -3;}
284 TBranch * branch = 0;
286 branch = tree->GetBranch(branchName);
287 if (!branch) {file->Close(); delete file; return -4;}
290 if (debugLevel==1 && tree->GetEntries()==0 ) return 1; //empty
292 tree->SetBranchStatus("*",1);
295 Int_t entries = tree->GetEntries();
296 for (Int_t i=0;i<entries; i++){
297 if (branch) branch->GetEntry(i);
298 else tree->GetEntry();
303 // never catched - as there is no exception in the ROOT IO
304 file->Close(); delete file;
308 file->Close(); delete file;
313 Bool_t AliXRDPROOFtoolkit::FilterList(const char*inputList, const char*fileList, Int_t checkLevel){
316 // inputList - list of original file names
317 // fileList - list of file to be checked
319 // - 1 - treeName (if * not checked)
322 // checkLevel - 0 - check only existance of the files and tree's +
323 // simple file corruption
324 // > 1 - check the content of the tree -
325 // (can crash as there do not exest exception handling in ROOT)
326 // Output - two streams are created - file with good entries
327 // "%s.Good a,d file with bad entries %s.Bad
329 // AliXRDPROOFtoolkit::FilterList("ppgrid2.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",1)
330 gEnv->SetValue("TFile.Recover", 0);
333 finput.open(inputList, ios_base::in);
336 focGood.open(Form("%s.Good",inputList), ios_base::out|ios_base::trunc);
337 focBad.open(Form("%s.Bad",inputList), ios_base::out|ios_base::trunc);
339 if(!finput.is_open()) {
340 cout<<"Can't open file "<<inputList<<endl;
344 // Read the input list of files and add them to the chain
346 TObjArray *array = (TString(fileList)).Tokenize(" ");
349 while(finput.good()) {
350 finput >> currentFile;
351 if (!currentFile.Contains("root")) continue; // protection
352 if (currentFile.Contains("alien://")){
353 focGood<<currentFile<<endl;
356 Bool_t isZip = currentFile.Contains("#");
357 const char * dirname = gSystem->DirName(currentFile.Data());
360 for (Int_t i=0; i<array->GetEntries(); i+=2){
363 snprintf(fname,1000, "%s/%s",dirname,array->At(i)->GetName());
364 if (((TObjString*)array->At(i))->String().Contains("*")){
365 snprintf(fname,1000, "%s", currentFile.Data());
369 const char * fileName = gSystem->BaseName(currentFile.Data());
370 TString fstring=fileName;
371 fstring[fstring.First("#")]=0;
372 snprintf(fname,1000, "%s/%s#%s",dirname,fstring.Data(),array->At(i)->GetName());
373 printf(fname, "To check %s%s#%s\n",dirname,fstring.Data(),array->At(i)->GetName());
376 printf("\nFile to be checked %s\n",fname);
377 //cout <<"\n arguments: "<< array->At(i+1)->GetName()<<" "<<checkLevel<<endl;
378 Int_t cstatus = CheckTreeInFile(fname, array->At(i+1)->GetName(), checkLevel,0);
379 //printf(" CheckTreeInFile returns %d",cstatus);
386 focGood<<currentFile<<endl;
388 focBad<<currentFile<<endl;
398 Bool_t AliXRDPROOFtoolkit::FilterListZip(const char*inputList, const char*fileList, Int_t checkLevel){
401 // inputList - list of original file names
402 // fileList - list of file to be checked
404 // - 1 - treeName (if * not checked)
407 // checkLevel - 0 - check only existance of the files and tree's +
408 // simple file corruption
409 // > 1 - check the content of the tree -
410 // (can crash as there do not exest exception handling in ROOT)
411 // Output - two streams are created - file with good entries
412 // "%s.Good a,d file with bad entries %s.Bad
414 // AliXRDPROOFtoolkit::FilterList("ppgrid2.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",1)
417 finput.open(inputList, ios_base::in);
420 focGood.open(Form("%s.Good",inputList), ios_base::out|ios_base::trunc);
421 focBad.open(Form("%s.Bad",inputList), ios_base::out|ios_base::trunc);
423 if(!finput.is_open()) {
424 cout<<"Can't open file "<<inputList<<endl;
428 // Read the input list of files and add them to the chain
430 TObjArray *array = (TString(fileList)).Tokenize(" ");
433 while(finput.good()) {
434 finput >> currentFile;
435 if (!currentFile.Contains("root")) continue; // protection
436 if (currentFile.Contains("alien://")){
437 focGood<<currentFile<<endl;
440 //Bool_t isZip = currentFile.Contains("#");
441 const char * dirname = gSystem->DirName(currentFile.Data());
442 const char * fileName = gSystem->BaseName(currentFile.Data());
443 TString fstring=fileName;
444 fstring[fstring.First("#")]=0;
446 for (Int_t i=0; i<array->GetEntries(); i+=2){
448 //if (isZip) sprintf(fname,
449 snprintf(fname,1000, "%s/%s#%s",dirname,fstring.Data(),array->At(i)->GetName());
450 printf(fname, "To check %s%s#%s\n",dirname,fstring.Data(),array->At(i)->GetName());
451 //cout <<"\n arguments: "<< array->At(i+1)->GetName()<<" "<<checkLevel<<endl;
452 Int_t cstatus = CheckTreeInFile(fname, array->At(i+1)->GetName(), checkLevel,0);
453 //printf(" CheckTreeInFile returns %d",cstatus);
460 focGood<<currentFile<<endl;
462 focBad<<currentFile<<endl;
474 Bool_t AliXRDPROOFtoolkit::XRDCopyDir(const char * idir, const char * files, const char *odir, Bool_t /*zip*/){
476 // idir - input directory
477 // odir - output directory
478 // files - the list of files to be coppied
479 // zip - not supported yet
483 // idir ="root://gsiaf.gsi.de:1094//sma/sim/v4-05-Rev-03/pp/0000";
484 // odir ="root://lxgrid2.gsi.de:1094//miranov/test/pp/0000";
485 // char *files="AliESDs.root AliESDfriend.root Kinematics.root";
487 TObjArray * array = str.Tokenize(" ");
488 Int_t nfiles = array->GetEntries();
492 for (Int_t ifile =0; ifile<nfiles; ifile++){
493 snprintf(infile,1000,"%s/%s", idir, array->At(ifile)->GetName());
494 snprintf(outfile,1000,"%s/%s", odir, array->At(ifile)->GetName());
495 printf("%s - %s\n",infile, outfile);
496 Bool_t result = TFile::Cp(infile,outfile);
505 void AliXRDPROOFtoolkit::JoinTreesIndex(const char * outputFile, const char * outputTree, const char *indexName, const char *inputTrees, Int_t debugLevel){
507 // Join together several tree according to the index
511 // outputFile : name of the output file
512 // outputTree : name of the output Tree
513 // indexName : name of the branch to be used as an index
515 // inputTrees : decription of the input trees setup
519 AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run","1#CPass1#run#tpcQA#TPCCPass1.root+1#VPass1#run#tpcQA#TPCVPass1.root+1#Pass1#run#tpcQA#TPCPass1.root+0#DAQ#run#joinTree#fproductionJoin.root+0#C#run#dcs#OCDBscan.root+0#CE#run#Fits#CEtrend.root");
521 Combine information form the Cpass1,VPass, and Pass1QA, calibration tree, DAQ information, trigger information
522 Make a File "outAll.root", with tree "joinAll", index of tree with name "run"
524 // Input tree configuration string:
526 const char *inputTrees="1#CPass1#run#tpcQA#TPCCPass1.root+1#VPass1#run#tpcQA#TPCVPass1.root+1#Pass1#run#tpcQA#TPCPass1.root+0#DAQ#run#joinTree#/home/miranov/test/dbQueries/fproductionJoin.root+0#C#run#dcs#OCDBscan.root+0#CE#run#Fits#CEtrend.root"
527 Describe 6 trees to be merged (string separated be +):
528 TObjArray *arrayInput = TString(inputTrees).Tokenize("+");
529 TObjString = 1#CPass1#run#tpcQA#TPCCPass1.root
530 TObjString = 1#VPass1#run#tpcQA#TPCVPass1.root
531 TObjString = 1#Pass1#run#tpcQA#TPCPass1.root
532 TObjString = 0#DAQ#run#joinTree#/home/miranov/test/dbQueries/fproductionJoin.root
533 TObjString = 0#C#run#dcs#OCDBscan.root
534 TObjString = 0#CE#run#Fits#CEtrend.root
536 Each tree is characterize by 5 parameters - separate by #
537 description="1#CPass1#run#tpcQA#TPCCPass1.root"
538 TString(description)->Tokenize("#").Print()
539 Collection name='TObjArray', class='TObjArray', size=16
540 TObjString = 1 ==> (0/1) index is used
541 TObjString = CPass1 ==> name of output branch in output tree
542 TObjString = run ==> name of the index
543 TObjString = tpcQA ==> name of the input tree in the input file
544 TObjString = TPCCPass1.root ==> name of the input file
550 TFile * fout = new TFile(outputFile,"recreate");
552 TTree *joinTree=new TTree(outputTree,outputTree);
554 // 1. Define setup. parse definition string
556 TObjArray *arrayInput = TString(inputTrees).Tokenize("+");
557 Int_t nTrees = arrayInput->GetEntries();
558 TObjArray * arrayFile = new TObjArray(nTrees); // array of TFiles with trees
559 TObjArray * arrayTrees = new TObjArray(nTrees); // array of trees
560 TObjArray * arrayNames = new TObjArray(nTrees); // name of tree
561 TObjArray * arrayRunID = new TObjArray(nTrees); // name of tree
562 TArrayI arrayEnableTree(nTrees);
563 for (Int_t i=0; i<2; i++) printf("\n");
564 printf("Joing query\n");
566 for (Int_t i=0; i<2; i++) printf("\n");
567 {for (Int_t itree=0; itree<nTrees; itree++){
569 TObjArray *description = TString(arrayInput->At(itree)->GetName()).Tokenize("#");
570 if (description->GetEntries()<4) {
571 printf("Fatal: Invalid description: %s\n", arrayInput->At(itree)->GetName());
574 TFile * f = TFile::Open(description->At(4)->GetName());
576 printf("Fatal: Invalid description: fileName %s\n", description->At(4)->GetName());
580 arrayFile->AddAt(f,itree);
581 TTree * tree = (TTree*)f->Get(description->At(3)->GetName());
583 printf("Fatal: Invalid description. Tree name\t%s\n", description->At(3)->GetName());
587 tree->SetCacheSize(400000000);
589 arrayTrees->AddAt(tree,itree);
591 arrayRunID->AddAt(new TObjString(description->At(2)->GetName()),itree);
592 arrayNames->AddAt(new TObjString(description->At(1)->GetName()),itree);
593 arrayEnableTree[itree]=atoi(description->At(0)->GetName());
598 // 2. Make the run list
601 map<int, int> runMap;
602 map<int, int> *runMapTree = new map<int, int>[nTrees];
603 //map<int, int> runMapTree[nTrees];
604 {for (Int_t itree=0; itree<nTrees; itree++){
605 TTree * tree = (TTree*)arrayTrees->At(itree);
606 Int_t entries=tree->GetEntries();
608 snprintf(query,2000,"%s:Entry$", arrayRunID->At(itree)->GetName());
609 entries = tree->Draw(query,"","goff");
610 for (Int_t ientry=0;ientry<entries; ientry++){
611 Int_t irun=Int_t(tree->GetV1()[ientry]);
612 // Int_t entryNr=Int_t(tree->GetV2()[ientry]);
613 if (arrayEnableTree[itree]>0) runMap[irun]+=1;
614 runMapTree[itree][irun]=ientry;
615 if (debugLevel>0) printf("%s\t%d\t%d\n",tree->GetName(), irun, runMapTree[itree][irun]);
624 joinTree->Branch(indexName, &jrun,Form("%s/I",indexName));
625 Int_t *status=new Int_t[nTrees];
626 char *brName = new char[10000];
627 char *brTitle= new char[10000];
630 {for (Int_t itree=0; itree<nTrees; itree++){
631 TTree * tree = (TTree*)arrayTrees->At(itree);
633 TString treeName=arrayNames->At(itree)->GetName();
634 if (treeName.Length()>0){
635 joinTree->Branch(Form("%s.status",treeName.Data()), &status[itree],Form("%s.status/I",treeName.Data()));
637 joinTree->Branch("status", &status[itree],"status/I");
640 Int_t nbranches= tree->GetListOfBranches()->GetEntries();
641 for (Int_t ibr=0; ibr<nbranches; ibr++){
642 TBranch * br = (TBranch*)(tree->GetListOfBranches()->At(ibr));
643 if (treeName.Length()>0){
644 sprintf(brName,"%s.%s",treeName.Data(), br->GetName());
645 sprintf(brTitle,"%s.%s",treeName.Data(), br->GetTitle());
647 sprintf(brName,"%s",br->GetName());
648 sprintf(brTitle,"%s",br->GetTitle());
651 TString className=br->GetClassName();
652 if (className.Length()==0){
653 TString str(br->GetTitle());
654 if (str[str.Length()-1]=='I') addr=new Int_t;
655 if (str[str.Length()-1]=='F') addr=new Float_t;
656 if (str[str.Length()-1]=='D') addr=new Double_t;
657 if (str[str.Length()-1]=='C') addr=new Char_t[10000];
658 if (addr) joinTree->Branch(brName, addr, brTitle);
659 br->SetAddress(addr);
661 TClass cclass(className);
662 TObject **addrClass = new TObject *;
664 printf("%s\t%s\n",br->GetName(), className.Data());
665 br->SetAddress(addrClass);
667 joinTree->Branch(brName,addrClass);
676 map<int, int>::iterator riter;
677 {for (riter=runMap.begin(); riter != runMap.end(); ++riter){
678 printf("%d\t%d\t", riter->first, riter->second);
680 for (Int_t itree=0; itree<nTrees; itree++){
681 TTree * tree = (TTree*)arrayTrees->At(itree);
682 Int_t entry= runMapTree[itree][jrun];
683 status[itree]=(entry>0)?1:0;
684 if (entry>=0) tree->GetEntry(entry);
685 printf("%d\t",entry);
692 joinTree->Write(outputTree);
699 void AliXRDPROOFtoolkit::CacheFileList(const char * fileIn, const char* cachePrefix){
701 // cache the list of file locally, cache valeus are stored in the location
702 // specified by optional argumen prefix
703 // 2 new files are created
704 // <fileIn>.cache - file with the location of cahe files
705 // <fileIn>.cacheLog - log file +list of files which can not be cached
708 fileIn = "TPCCPass1.list";
714 fout.open(Form("%s.cache",fileIn));
716 foutLog.open(Form("%s.cacheLog",fileIn));
717 // Read the input list of files and add them to the chain
726 fname.ReplaceAll("-","_");
727 fname.ReplaceAll("/","_");
728 fname.ReplaceAll(":","_");
729 cacheFile=cachePrefix;
731 printf("%s\t%s\n",currentFile.Data(),cacheFile.Data());
732 if (TFile::Cp(currentFile.Data(),cacheFile.Data())){
733 fout<<cacheFile.Data()<<"\n";
734 foutLog<<s.AsString();
735 foutLog<<cacheFile.Data()<<"n";
737 foutLog<<"Copy failed"<<currentFile.Data()<<cacheFile.Data()<<"\n";
746 void AliXRDPROOFtoolkit::MakeTreeFromList(const char *fout, const char * treeOut, const char * treeIn, const char * flist, Bool_t debug){
748 // join trees from the list and make a common tree - stored in the file
752 const char * fout="TPCCpass1.root";
753 const char *treeOut="tpcQA"
754 const char *treeIn="tpcQA"
755 const char * flist="TPCCPass1.list"
758 printf("MakeTreeFromList\n");
759 printf("fout=%s\n",fout);
760 printf("treeOut=%s\n",treeOut);
761 printf("treeIn=%s\n",treeIn);
762 printf("fileList=%s\n",flist);
767 foutLog.open(Form("%s.chainLog",flist));
768 // Read the input list of files and add them to the chain
774 TFile * f = TFile::Open(currentFile.Data());
775 foutLog<<"Opening file"<<currentFile.Data();
777 foutLog<<"Error opening file\t"<<currentFile<<"\n";
778 cout<<"Error opening file\t"<<currentFile<<"\n";
781 TTree * tree = (TTree*)f->Get(treeIn);
783 foutLog<<"Error opening tree\t"<<currentFile<<treeIn<<"\n";
784 cout<<"Error opening tree\t"<<currentFile<<treeIn<<"\n";
788 if (tree->GetListOfBranches()==0){
789 foutLog<<"Error opening tree\t"<<currentFile<<treeIn<<"\n";
790 cout<<"Error opening tree\t"<<currentFile<<treeIn<<"\n";
793 Int_t nbranchesCurrent = tree->GetListOfBranches()->GetEntries();
794 if ( nbranches ==0 ) nbranches=nbranchesCurrent;
795 if ( nbranches!=nbranchesCurrent){
796 foutLog<<"Error tree layout\t"<<currentFile<<treeIn<<nbranches<<nbranchesCurrent<<"\n";
797 cout<<"Error tree layout\t" <<currentFile<<treeIn<<nbranches<<nbranchesCurrent<<"\n";
802 foutLog<<"Number of files"<<counter<<"\n";
803 cout<< "Number of files"<<counter<<"\n";
806 TChain * chain = AliXRDPROOFtoolkit::MakeChain(flist,treeIn,0,1000000000,0);
808 if (!chain) status=kFALSE;
809 if (chain->GetEntries()==0) status=kFALSE;
811 printf("Incorrect list (%s) or trees (%s)", flist,treeIn);
814 TFile *fileOut= TFile::Open(fout, "recreate");
815 TTree * tree = chain->CopyTree("1");
817 tree->Write(treeOut);