1 /**************************************************************************
2 * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
19 //TOOLKIT for chain manipulation:
22 // 1. Check the list of files
23 AliXRDPROOFtoolkit toolkit;
24 AliXRDPROOFtoolkit::FilterList("pp.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",0)
25 AliXRDPROOFtoolkit::FilterList("pp.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",1)
27 //2. make a chain or random chain form the list of files
28 TChain * chain = toolkit.MakeChain("esd.txt","esdTree",0,10)
29 TChain * chainRandom = toolkit.MakeChainrandom("esd.txt","esdTree",0,10)
30 chain->Draw("fTPCnclsF");
37 #include <TObjArray.h>
38 #include <TObjString.h>
50 #include <AliXRDPROOFtoolkit.h>
53 ClassImp(AliXRDPROOFtoolkit)
57 //______________________________________________________________________________
58 AliXRDPROOFtoolkit::AliXRDPROOFtoolkit () :
60 fVerbose(kFALSE), // verbso mode - print command
61 fUserName(""), // user name
62 fUserGroup(0) // user group info
67 fUserGroup = gSystem->GetUserInfo();
68 fUserName = fUserGroup->fUser;
76 TChain* AliXRDPROOFtoolkit::MakeChain(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles, Int_t startFile)
81 TChain* chain = new TChain(treeName);
83 // Open the input stream
87 // Read the input list of files and add them to the chain
96 if (!currentFile.Contains("root")) continue; // protection
98 if (counter<startFile) continue;
99 if (counter>maxFiles+startFile) break;
100 TFile * f = TFile::Open(currentFile.Data());
102 chain->Add(currentFile.Data());
112 TChain* AliXRDPROOFtoolkit::MakeChainRandom(const char*fileIn, const char * treeName,const char *fName, Int_t maxFiles, Int_t startFile)
115 // Create a TDSet - files are in random order
117 // filein - input list text file
118 // treename - containg tree
119 // maxFiles - maximum number of files included
121 TObjArray array(10000);
123 TChain* chain = new TChain(treeName);
125 // Open the input stream
129 // Read the input list of files and add them to the chain
138 if (!currentFile.Contains("root")) continue; // protection
140 // chain->Add(currentFile.Data());
141 array.AddLast(new TObjString(currentFile));
144 Int_t entries = array.GetEntries();
145 printf("Number of entries\t%d\n",entries);
149 Double_t *randomI = new Double_t[entries];
150 Int_t *indexes = new Int_t[entries];
151 for (Int_t i=0;i<entries; i++) randomI[i]=gRandom->Rndm();
152 TMath::Sort(entries,randomI,indexes);
154 for (Int_t i=startFile; (i<startFile+maxFiles) && (i<entries); i++){
155 Int_t ifile = indexes[i];
156 if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
157 printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
158 chain->Add(array.At(ifile)->GetName());
159 array.At(ifile)->SetBit(TObject::kCannotPick);
167 TDSet* AliXRDPROOFtoolkit::MakeSet(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
170 // Create the TDSet out of list
171 // filein - input list text file
172 // treename - containg tree
173 // maxFiles - maximum number of files included
175 TDSet* chain = new TDSet(treeName);
177 // Open the input stream
181 // Read the input list of files and add them to the chain
190 if (!currentFile.Contains("root")) continue; // protection
192 if (maxFiles>0 && counter>maxFiles) break;
193 chain->Add(currentFile.Data());
202 TDSet* AliXRDPROOFtoolkit::MakeSetRandom(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
205 // Create a TDSet - files are in random order
207 // filein - input list text file
208 // treename - containg tree
209 // maxFiles - maximum number of files included
211 TObjArray array(10000);
213 TDSet* chain = new TDSet(treeName);
215 // Open the input stream
219 // Read the input list of files and add them to the chain
228 if (!currentFile.Contains("root")) continue; // protection
230 // chain->Add(currentFile.Data());
231 array.AddLast(new TObjString(currentFile));
234 Int_t entries = array.GetEntries();
235 printf("Number of entries\t%d",entries);
236 if (maxFiles<0) maxFiles=entries;
237 if (maxFiles>entries) maxFiles=entries;
238 for (Int_t i=0; i<maxFiles; i++){
239 Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
240 if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
241 printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
242 chain->Add(array.At(ifile)->GetName());
243 array.At(ifile)->SetBit(TObject::kCannotPick);
258 Int_t AliXRDPROOFtoolkit::CheckTreeInFile(const char*fileName,const char*treeName, Int_t debugLevel, const char *branchName){
260 // Check the tree in file
261 // fileName - the name of the file with tree
262 // treeName - the name of file
263 // debugLevel - 0 check the existance of the file - 1 make loop over entries
264 // branchName - if debugLevel>0 the branch is chcecked
265 // if brnachName =0 the content of full tree is chcecked
266 // return value = 0 - Check things OK
267 // -1 - file not exist or not accesible
268 // -2 - file is zombie
269 // -3 - tree not present
270 // -4 - branch not present
271 TFile * file = TFile::Open(fileName);
272 if (!file) { return -1;}
273 if (file->IsZombie()) {file->Close(); delete file; return -2;};
275 TString TrName(treeName);
277 //cout <<" treename ==== *"<<endl;;
278 file->Close(); delete file;
281 TTree * tree = (TTree*)file->Get(treeName);
282 if (!tree) {file->Close(); delete file; return -3;}
283 TBranch * branch = 0;
285 branch = tree->GetBranch(branchName);
286 if (!branch) {file->Close(); delete file; return -4;}
289 if (debugLevel==1 && tree->GetEntries()==0 ) return 1; //empty
291 tree->SetBranchStatus("*",1);
294 Int_t entries = tree->GetEntries();
295 for (Int_t i=0;i<entries; i++){
296 if (branch) branch->GetEntry(i);
297 else tree->GetEntry();
302 // never catched - as there is no exception in the ROOT IO
303 file->Close(); delete file;
307 file->Close(); delete file;
312 Bool_t AliXRDPROOFtoolkit::FilterList(const char*inputList, const char*fileList, Int_t checkLevel){
315 // inputList - list of original file names
316 // fileList - list of file to be checked
318 // - 1 - treeName (if * not checked)
321 // checkLevel - 0 - check only existance of the files and tree's +
322 // simple file corruption
323 // > 1 - check the content of the tree -
324 // (can crash as there do not exest exception handling in ROOT)
325 // Output - two streams are created - file with good entries
326 // "%s.Good a,d file with bad entries %s.Bad
328 // AliXRDPROOFtoolkit::FilterList("ppgrid2.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",1)
329 gEnv->SetValue("TFile.Recover", 0);
332 finput.open(inputList, ios_base::in);
335 focGood.open(Form("%s.Good",inputList), ios_base::out|ios_base::trunc);
336 focBad.open(Form("%s.Bad",inputList), ios_base::out|ios_base::trunc);
338 if(!finput.is_open()) {
339 cout<<"Can't open file "<<inputList<<endl;
343 // Read the input list of files and add them to the chain
345 TObjArray *array = (TString(fileList)).Tokenize(" ");
348 while(finput.good()) {
349 finput >> currentFile;
350 if (!currentFile.Contains("root")) continue; // protection
351 if (currentFile.Contains("alien://")){
352 focGood<<currentFile<<endl;
355 Bool_t isZip = currentFile.Contains("#");
356 const char * dirname = gSystem->DirName(currentFile.Data());
359 for (Int_t i=0; i<array->GetEntries(); i+=2){
362 snprintf(fname,1000, "%s/%s",dirname,array->At(i)->GetName());
363 if (((TObjString*)array->At(i))->String().Contains("*")){
364 snprintf(fname,1000, "%s", currentFile.Data());
368 const char * fileName = gSystem->BaseName(currentFile.Data());
369 TString fstring=fileName;
370 fstring[fstring.First("#")]=0;
371 snprintf(fname,1000, "%s/%s#%s",dirname,fstring.Data(),array->At(i)->GetName());
372 printf(fname, "To check %s%s#%s\n",dirname,fstring.Data(),array->At(i)->GetName());
375 printf("\nFile to be checked %s\n",fname);
376 //cout <<"\n arguments: "<< array->At(i+1)->GetName()<<" "<<checkLevel<<endl;
377 Int_t cstatus = CheckTreeInFile(fname, array->At(i+1)->GetName(), checkLevel,0);
378 //printf(" CheckTreeInFile returns %d",cstatus);
385 focGood<<currentFile<<endl;
387 focBad<<currentFile<<endl;
396 Bool_t AliXRDPROOFtoolkit::FilterListZip(const char*inputList, const char*fileList, Int_t checkLevel){
399 // inputList - list of original file names
400 // fileList - list of file to be checked
402 // - 1 - treeName (if * not checked)
405 // checkLevel - 0 - check only existance of the files and tree's +
406 // simple file corruption
407 // > 1 - check the content of the tree -
408 // (can crash as there do not exest exception handling in ROOT)
409 // Output - two streams are created - file with good entries
410 // "%s.Good a,d file with bad entries %s.Bad
412 // AliXRDPROOFtoolkit::FilterList("ppgrid2.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",1)
415 finput.open(inputList, ios_base::in);
418 focGood.open(Form("%s.Good",inputList), ios_base::out|ios_base::trunc);
419 focBad.open(Form("%s.Bad",inputList), ios_base::out|ios_base::trunc);
421 if(!finput.is_open()) {
422 cout<<"Can't open file "<<inputList<<endl;
426 // Read the input list of files and add them to the chain
428 TObjArray *array = (TString(fileList)).Tokenize(" ");
431 while(finput.good()) {
432 finput >> currentFile;
433 if (!currentFile.Contains("root")) continue; // protection
434 if (currentFile.Contains("alien://")){
435 focGood<<currentFile<<endl;
438 //Bool_t isZip = currentFile.Contains("#");
439 const char * dirname = gSystem->DirName(currentFile.Data());
440 const char * fileName = gSystem->BaseName(currentFile.Data());
441 TString fstring=fileName;
442 fstring[fstring.First("#")]=0;
444 for (Int_t i=0; i<array->GetEntries(); i+=2){
446 //if (isZip) sprintf(fname,
447 snprintf(fname,1000, "%s/%s#%s",dirname,fstring.Data(),array->At(i)->GetName());
448 printf(fname, "To check %s%s#%s\n",dirname,fstring.Data(),array->At(i)->GetName());
449 //cout <<"\n arguments: "<< array->At(i+1)->GetName()<<" "<<checkLevel<<endl;
450 Int_t cstatus = CheckTreeInFile(fname, array->At(i+1)->GetName(), checkLevel,0);
451 //printf(" CheckTreeInFile returns %d",cstatus);
458 focGood<<currentFile<<endl;
460 focBad<<currentFile<<endl;
472 Bool_t AliXRDPROOFtoolkit::XRDCopyDir(const char * idir, const char * files, const char *odir, Bool_t /*zip*/){
474 // idir - input directory
475 // odir - output directory
476 // files - the list of files to be coppied
477 // zip - not supported yet
481 // idir ="root://gsiaf.gsi.de:1094//sma/sim/v4-05-Rev-03/pp/0000";
482 // odir ="root://lxgrid2.gsi.de:1094//miranov/test/pp/0000";
483 // char *files="AliESDs.root AliESDfriend.root Kinematics.root";
485 TObjArray * array = str.Tokenize(" ");
486 Int_t nfiles = array->GetEntries();
490 for (Int_t ifile =0; ifile<nfiles; ifile++){
491 snprintf(infile,1000,"%s/%s", idir, array->At(ifile)->GetName());
492 snprintf(outfile,1000,"%s/%s", odir, array->At(ifile)->GetName());
493 printf("%s - %s\n",infile, outfile);
494 Bool_t result = TFile::Cp(infile,outfile);
502 void AliXRDPROOFtoolkit::JoinTreesIndex(const char * outputFile, const char * outputTree, const char *indexName, const char *inputTrees, Int_t debugLevel){
504 // Join together several tree according to the index
508 // outputFile : name of the output file
509 // outputTree : name of the output Tree
510 // indexName : name of the branch to be used as an index
512 // inputTrees : decription of the input trees setup
516 AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run","1#CPass1#run#tpcQA#TPCCPass1.root+1#VPass1#run#tpcQA#TPCVPass1.root+1#Pass1#run#tpcQA#TPCPass1.root+0#DAQ#run#joinTree#fproductionJoin.root+0#C#run#dcs#OCDBscan.root+0#CE#run#Fits#CEtrend.root");
518 Combine information form the Cpass1,VPass, and Pass1QA, calibration tree, DAQ information, trigger information
519 Make a File "outAll.root", with tree "joinAll", index of tree with name "run"
521 // Input tree configuration string:
523 const char *inputTrees="1#CPass1#run#tpcQA#TPCCPass1.root+1#VPass1#run#tpcQA#TPCVPass1.root+1#Pass1#run#tpcQA#TPCPass1.root+0#DAQ#run#joinTree#/home/miranov/test/dbQueries/fproductionJoin.root+0#C#run#dcs#OCDBscan.root+0#CE#run#Fits#CEtrend.root"
524 Describe 6 trees to be merged (string separated be +):
525 TObjArray *arrayInput = TString(inputTrees).Tokenize("+");
526 TObjString = 1#CPass1#run#tpcQA#TPCCPass1.root
527 TObjString = 1#VPass1#run#tpcQA#TPCVPass1.root
528 TObjString = 1#Pass1#run#tpcQA#TPCPass1.root
529 TObjString = 0#DAQ#run#joinTree#/home/miranov/test/dbQueries/fproductionJoin.root
530 TObjString = 0#C#run#dcs#OCDBscan.root
531 TObjString = 0#CE#run#Fits#CEtrend.root
533 Each tree is characterize by 5 parameters - separate by #
534 description="1#CPass1#run#tpcQA#TPCCPass1.root"
535 TString(description)->Tokenize("#").Print()
536 Collection name='TObjArray', class='TObjArray', size=16
537 TObjString = 1 ==> (0/1) index is used
538 TObjString = CPass1 ==> name of output branch in output tree
539 TObjString = run ==> name of the index
540 TObjString = tpcQA ==> name of the input tree in the input file
541 TObjString = TPCCPass1.root ==> name of the input file
547 TFile * fout = new TFile(outputFile,"recreate");
549 TTree *joinTree=new TTree(outputTree,outputTree);
551 // 1. Define setup. parse definition string
553 TObjArray *arrayInput = TString(inputTrees).Tokenize("+");
554 Int_t nTrees = arrayInput->GetEntries();
555 TObjArray * arrayFile = new TObjArray(nTrees); // array of TFiles with trees
556 TObjArray * arrayTrees = new TObjArray(nTrees); // array of trees
557 TObjArray * arrayNames = new TObjArray(nTrees); // name of tree
558 TObjArray * arrayRunID = new TObjArray(nTrees); // name of tree
559 TArrayI arrayEnableTree(nTrees);
560 for (Int_t i=0; i<2; i++) printf("\n");
561 printf("Joing query\n");
563 for (Int_t i=0; i<2; i++) printf("\n");
564 {for (Int_t itree=0; itree<nTrees; itree++){
566 TObjArray *description = TString(arrayInput->At(itree)->GetName()).Tokenize("#");
567 if (description->GetEntries()<4) {
568 printf("Fatal: Invalid description: %s\n", arrayInput->At(itree)->GetName());
571 TFile * f = TFile::Open(description->At(4)->GetName());
573 printf("Fatal: Invalid description: fileName %s\n", description->At(4)->GetName());
576 arrayFile->AddAt(f,itree);
577 TTree * tree = (TTree*)f->Get(description->At(3)->GetName());
579 printf("Fatal: Invalid description. Tree name\t%s\n", description->At(3)->GetName());
582 tree->SetCacheSize(400000000);
584 arrayTrees->AddAt(tree,itree);
586 arrayRunID->AddAt(new TObjString(description->At(2)->GetName()),itree);
587 arrayNames->AddAt(new TObjString(description->At(1)->GetName()),itree);
588 arrayEnableTree[itree]=atoi(description->At(0)->GetName());
592 // 2. Make the run list
595 map<int, int> runMap;
596 map<int, int> *runMapTree = new map<int, int>[nTrees];
597 //map<int, int> runMapTree[nTrees];
598 {for (Int_t itree=0; itree<nTrees; itree++){
599 TTree * tree = (TTree*)arrayTrees->At(itree);
600 Int_t entries=tree->GetEntries();
602 snprintf(query,2000,"%s:Entry$", arrayRunID->At(itree)->GetName());
603 entries = tree->Draw(query,"","goff");
604 for (Int_t ientry=0;ientry<entries; ientry++){
605 Int_t irun=Int_t(tree->GetV1()[ientry]);
606 // Int_t entryNr=Int_t(tree->GetV2()[ientry]);
607 if (arrayEnableTree[itree]>0) runMap[irun]+=1;
608 runMapTree[itree][irun]=ientry;
609 if (debugLevel>0) printf("%s\t%d\t%d\n",tree->GetName(), irun, runMapTree[itree][irun]);
617 joinTree->Branch(indexName, &jrun,Form("%s/I",indexName));
618 Int_t *status=new Int_t[nTrees];
619 char *brName = new char[10000];
620 char *brTitle= new char[10000];
623 {for (Int_t itree=0; itree<nTrees; itree++){
624 TTree * tree = (TTree*)arrayTrees->At(itree);
626 TString treeName=arrayNames->At(itree)->GetName();
627 if (treeName.Length()>0){
628 joinTree->Branch(Form("%s.status",treeName.Data()), &status[itree],Form("%s.status/I",treeName.Data()));
630 joinTree->Branch("status", &status[itree],"status/I");
633 Int_t nbranches= tree->GetListOfBranches()->GetEntries();
634 for (Int_t ibr=0; ibr<nbranches; ibr++){
635 TBranch * br = (TBranch*)(tree->GetListOfBranches()->At(ibr));
636 if (treeName.Length()>0){
637 sprintf(brName,"%s.%s",treeName.Data(), br->GetName());
638 sprintf(brTitle,"%s.%s",treeName.Data(), br->GetTitle());
640 sprintf(brName,"%s",br->GetName());
641 sprintf(brTitle,"%s",br->GetTitle());
644 TString className=br->GetClassName();
645 if (className.Length()==0){
646 TString str(br->GetTitle());
647 if (str[str.Length()-1]=='I') addr=new Int_t;
648 if (str[str.Length()-1]=='F') addr=new Float_t;
649 if (str[str.Length()-1]=='D') addr=new Double_t;
650 if (str[str.Length()-1]=='C') addr=new Char_t[10000];
651 if (addr) joinTree->Branch(brName, addr, brTitle);
652 br->SetAddress(addr);
654 TClass cclass(className);
655 TObject **addrClass = new TObject *;
657 printf("%s\t%s\n",br->GetName(), className.Data());
658 br->SetAddress(addrClass);
660 joinTree->Branch(brName,addrClass);
669 map<int, int>::iterator riter;
670 {for (riter=runMap.begin(); riter != runMap.end(); ++riter){
671 printf("%d\t%d\t", riter->first, riter->second);
673 for (Int_t itree=0; itree<nTrees; itree++){
674 TTree * tree = (TTree*)arrayTrees->At(itree);
675 Int_t entry= runMapTree[itree][jrun];
676 status[itree]=(entry>0)?1:0;
677 if (entry>=0) tree->GetEntry(entry);
678 printf("%d\t",entry);
684 joinTree->Write("joinTree");