2 TTOOLKIT to acces files on XRD
8 gSystem->AddIncludePath("-I$ALICE_ROOT/TPC/macros")
9 gROOT->LoadMacro("$ALICE_ROOT/TPC/macros/AliXRDPROOFtoolkit.cxx+")
12 Retrieve the list of files : POSSIBLE ONLY FOR USER WITH LOGIN ACCESS TO THE XRD MACHINES
14 AliXRDPROOFtoolkit tool;
15 tool.ListOfFiles("all.txt","/data.local2/sma/sim/v4-06-Rev-03/pp*", "AliESDs.root", kFALSE);
16 tool.ListOfFiles("pp.txt","/data.local2/sma/sim/v4-06-Rev-03/pp", "AliESDs.root", kFALSE);
18 tool.ListOfFiles("pp64.txt","/data.local2/sma/sim/v4-06-Rev-03/pp64", "AliESDs.root", kFALSE);
19 tool.ListOfFiles("pp64m.txt","/data.local2/sma/sim/v4-06-Rev-03/pp64", "AliESDs.root", kTRUE);
22 AliXRDPROOFtoolkit::FilterList("pp.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",0)
23 AliXRDPROOFtoolkit::FilterList("pp.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",1)
27 AliXRDPROOFtoolkit tool;
28 TChain * chain = tool.MakeChain("pp.txt","esdTree",0,10)
29 chain->Draw("fTPCnclsF");
32 3. Process logs - ONLY priority users - with ssh acces
33 AliXRDPROOFtoolkit tool;
34 tool.FilterSegFault();
35 TTree * treeSys = tool.DumpSys(kTRUE)
42 #include <TObjArray.h>
43 #include <TObjString.h>
57 #include <AliXRDPROOFtoolkit.h>
60 ClassImp(AliXRDPROOFtoolkit);
64 //______________________________________________________________________________
65 AliXRDPROOFtoolkit::AliXRDPROOFtoolkit () : TObject ()
68 // Default -GSI specific setup
70 for(Int_t i=255;i<269;i++)
71 //for(Int_t i=259;i<260;i++)
72 listeMachine.push_back(new TString(Form("lxb%d.gsi.de", i)));
73 fUserGroup = gSystem->GetUserInfo();
74 fUserName = fUserGroup->fUser;
78 void AliXRDPROOFtoolkit::Print(Option_t* /*option*/) const{
80 // Print current setup of toolkit
82 printf("User name\t%s\n", fUserName.Data());
83 printf("List of slaves\n");
85 for(UInt_t i=0;i<listeMachine.size();i++)
86 printf("%s\n",listeMachine[i]->Data());
90 //______________________________________________________________________________
91 void AliXRDPROOFtoolkit::AddMachine (const char*name)
94 // add computer to the list
96 listeMachine.push_back( new TString(name));
100 //______________________________________________________________________________
101 Bool_t AliXRDPROOFtoolkit::ListOfFiles(const char*fileName, const char*path, const char*filter, Bool_t displayMachine)
104 // Get the list of files on "registerd slaves"
106 // fileName - Resultinfg file with list
107 // path - starting path on slave e.g /data.local2/sma/
108 // filter - filter expression e.g. AliESDs.root
109 // display machine - not used yet
114 // FILTER expression - to get REDIRECTOR NAME
116 //what we want for sed
117 // sprintf(filterXRD,"sed s/\\/data.local2/root:\\/\\/gsiaf.gsi.de:1094\\//");
118 //what we should write
119 sprintf(filterXRD,"sed s/\\\\/data.local2/root:\\\\/\\\\/gsiaf.gsi.de:1094\\\\//");
123 gSystem->Exec(Form("echo >%s",fileName));
124 for(UInt_t i=0; i<listeMachine.size(); i++){
126 sprintf(filterXRD,"sed s/\\\\/data.local2/root:\\\\/\\\\/%s:1094\\\\//",listeMachine[i]->Data());
128 cout<<"Inspecting "<<listeMachine[i]->Data()<<" ..."<<endl;
129 sprintf(command,"lsrun -m %s find %s | grep %s | %s >> %s", listeMachine[i]->Data(), path, filter, filterXRD,fileName);
131 gSystem->Exec(command);
137 TChain* AliXRDPROOFtoolkit::MakeChain(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles, Int_t startFile)
142 TChain* chain = new TChain(treeName);
144 // Open the input stream
148 // Read the input list of files and add them to the chain
157 if (!currentFile.Contains("root")) continue; // protection
159 if (counter<startFile) continue;
160 if (counter>maxFiles+startFile) break;
161 chain->Add(currentFile.Data());
169 TChain* AliXRDPROOFtoolkit::MakeChainRandom(const char*fileIn, const char * treeName,const char *fName, Int_t maxFiles, Int_t startFile)
172 // Create a TDSet - files are in random order
174 // filein - input list text file
175 // treename - containg tree
176 // maxFiles - maximum number of files included
178 TObjArray array(10000);
180 TChain* chain = new TChain(treeName);
182 // Open the input stream
186 // Read the input list of files and add them to the chain
195 if (!currentFile.Contains("root")) continue; // protection
197 // chain->Add(currentFile.Data());
198 array.AddLast(new TObjString(currentFile));
201 Int_t entries = array.GetEntries();
202 printf("Number of entries\t%d",entries);
203 if (maxFiles<0) maxFiles=entries;
204 if (maxFiles>entries) maxFiles=entries;
208 for (Int_t i=0; i<maxFiles; i++){
209 Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
210 if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
211 printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
212 chain->Add(array.At(ifile)->GetName());
213 array.At(ifile)->SetBit(TObject::kCannotPick);
221 TDSet* AliXRDPROOFtoolkit::MakeSet(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
224 // Create the TDSet out of list
225 // filein - input list text file
226 // treename - containg tree
227 // maxFiles - maximum number of files included
229 TDSet* chain = new TDSet(treeName);
231 // Open the input stream
235 // Read the input list of files and add them to the chain
244 if (!currentFile.Contains("root")) continue; // protection
246 if (maxFiles>0 && counter>maxFiles) break;
247 chain->Add(currentFile.Data());
256 TDSet* AliXRDPROOFtoolkit::MakeSetRandom(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
259 // Create a TDSet - files are in random order
261 // filein - input list text file
262 // treename - containg tree
263 // maxFiles - maximum number of files included
265 TObjArray array(10000);
267 TDSet* chain = new TDSet(treeName);
269 // Open the input stream
273 // Read the input list of files and add them to the chain
282 if (!currentFile.Contains("root")) continue; // protection
284 // chain->Add(currentFile.Data());
285 array.AddLast(new TObjString(currentFile));
288 Int_t entries = array.GetEntries();
289 printf("Number of entries\t%d",entries);
290 if (maxFiles<0) maxFiles=entries;
291 if (maxFiles>entries) maxFiles=entries;
292 for (Int_t i=0; i<maxFiles; i++){
293 Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
294 if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
295 printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
296 chain->Add(array.At(ifile)->GetName());
297 array.At(ifile)->SetBit(TObject::kCannotPick);
306 //_______________________________________________________________________________
309 void AliXRDPROOFtoolkit::FilterSegFault(const char *filter){
311 // Print the list of slaves finished with seg fault
313 printf("List of the PROOF slaves with seg fault (Last session) \n");
314 for(UInt_t i=0; i<listeMachine.size(); i++){
315 if (HasSegFault(listeMachine[i]->Data(), filter)){
316 printf("%s\n",listeMachine[i]->Data());
321 //_______________________________________________________________________________
323 Bool_t AliXRDPROOFtoolkit::HasSegFault(const char * machine, const char *filter){
325 // check if segmentation fault on the node
329 char *path = "segfault.log";
330 Long_t id,size,flags,modtime;
332 // first check if we have a log file
333 sprintf(command0, Form("lsrun -m %s find /data.local2/proof/%s/ | grep %s | grep .log >filelist.list", machine,fUserName.Data(), filter));
335 printf("%s\n",command0);
337 gSystem->Exec(command0);
338 gSystem->GetPathInfo("filelist.list", &id, &size, &flags, &modtime);
339 if (size<1) return kFALSE;
342 // now check the content
344 sprintf(command, Form("lsrun -m %s cat `lsrun -m %s find /data.local2/proof/%s/ | grep %s|grep log ` | grep segmentation >segfault.log", machine, machine,fUserName.Data(), filter));
347 printf("%s\n",command);
349 if (fVerbose>1) gSystem->Exec(command);
350 gSystem->GetPathInfo(path, &id, &size, &flags, &modtime);
355 //--------------------------------------------------------------------
357 TTree * AliXRDPROOFtoolkit::DumpSys(Bool_t verbose){
359 // Dump memory usage on worker nodes
360 gSystem->Exec("echo > syswatch.log");
361 for(UInt_t i=0; i<listeMachine.size(); i++){
362 if (verbose) gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data()));
363 gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log >> syswatch.log",listeMachine[i]->Data()));
366 TTree * tree = new TTree("asci","asci");
367 tree->SetDirectory(0);
368 // tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F");
369 tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F");
373 // //--------------------------------------------------------------------
375 TTree * AliXRDPROOFtoolkit::DumpSys2(Bool_t verbose){
377 // Dump memory usage on worker nodes
379 gSystem->Exec("echo > syswatch.log");
380 for(UInt_t i=0; i<listeMachine.size(); i++){
382 gSystem->Exec(Form("lsrun -m %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data()));
383 gSystem->Exec(Form("lsrun -m %s cat /data.local2/proof/miranov/last-workersession/syswatch.log >> syswatch.log",listeMachine[i]->Data()));
386 TTree * tree = new TTree("asci","asci");
387 tree->SetDirectory(0);
388 // tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F");
389 tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F");
399 TTree* AliXRDPROOFtoolkit::DumpFiles(Bool_t verbose){
403 gSystem->Exec("touch filewatch.log");
404 for(UInt_t i=0; i<listeMachine.size(); i++){
405 if (verbose) gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/worker*/filewatch.log >>filewatch.log",listeMachine[i]->Data()));
406 gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/worker*/filewatch.log >>filewatch.log",listeMachine[i]->Data()));
408 TTree * tree = new TTree("asci","asci");
409 tree->SetDirectory(0);
410 tree->ReadFile("filewatch.log","hname/C:fname/C:entry/I:status/I");
419 //______________________________________________________________________________
420 void AliXRDPROOFtoolkit::CheckFiles (const char*fileIn, UInt_t checkLevel, const char*treeToRetrieve, const char*varexp, const char*selection)
426 fic.open(fileIn, ios_base::in);
429 focGood.open(Form("%s.Good",fileIn), ios_base::out|ios_base::trunc);
430 focBad.open(Form("%s.Bad",fileIn), ios_base::out|ios_base::trunc);
433 cout<<"Can't open file "<<fileIn<<endl;
444 Float_t compressionFactor;
447 TObjString * fileName=0x0;
448 TObjString * machineName=0x0;
449 // TChain chain ("check_chain");
451 TFile fout(Form("%s.root",fileIn),"recreate");
452 TTree * tree=new TTree("stats", "stats of AliXRDPROOFtoolkit::CheckFiles function");
454 tree->Branch ("machine", "TObjString", &machineName, 256000,0);
455 tree->Branch ("file", "TObjString", &fileName, 256000,0);
456 tree->Branch ("level", &level, "level/I");
457 tree->Branch ("size", &size, "size/L");
458 tree->Branch ("nkey", &nkey, "nkey/I");
459 tree->Branch ("compress", &compressionFactor, "compress/F");
460 tree->Branch ("version", &version, "version/I");
462 // start loop over the files
463 fic.getline(buffer, sizeof(buffer));
466 // get the machine name if present in the file
467 machine=strchr(buffer, '\t');
472 machineName=new TObjString(machine);
474 machineName=new TObjString("x");
477 cout<<"Inspecting file :"<<line<<endl;
481 fileName=new TObjString(line);
483 TFile * f=TFile::Open(line);
484 // chain.AddFile(line);
488 compressionFactor=-1;
491 if (fileName->String().Contains("AliESDs.root")){
495 char fnameFriend[1000];
496 sprintf(fnameFriend,"%s/AliESDfriends.root",gSystem->DirName(fileName->String().Data()));
497 cout<<"Inspecting file :"<<fnameFriend<<endl;
498 TFile * ffriend=TFile::Open(fnameFriend);
499 if (!ffriend) level=-4;
501 if (ffriend->IsZombie()) level=-3;
502 if (ffriend->TestBit(TFile::kRecovered) || ffriend->TestBit(TFile::kWriteError)){
511 if(level>=-1 && f!=0x0){
515 compressionFactor=f->GetCompressionFactor();
516 version=f->GetVersion();
518 if(checkLevel>0 && !f->IsZombie()){
520 if(checkLevel>1 && treeToRetrieve!="" && (getTree=(TTree*)f->Get(treeToRetrieve))!=0x0){
522 Int_t tentries = getTree->GetEntries();
523 if (tentries>=0) level=4;
524 cout<<"Number of entries :"<<getTree->GetEntries()<<endl;
526 if(checkLevel>3 && varexp!="" &&tentries>0) {
527 getTree->SetBranchStatus("*",1);
529 TH1F his("his","his",100,-1,1);
530 Int_t selected = getTree->Draw(Form("%s>>his",varexp), selection, "goff", 1, tentries-1);
531 cout<<"Selected\t"<<selected<<endl;
535 //try to remove the created histogrames ...
536 // TH1F *htemp = (TH1F*)gPad->GetPrimitive("htemp"); // 1D
537 // TGraph *graph = (TGraph*)gPad->GetPrimitive("Graph"); // 2D
538 // if(htemp!=0x0) {cout<<"removing TH1D"<<endl; delete htemp;}
539 // else if(graph!=0x0) {cout<<"removing TGraph"<<endl; delete graph;}
540 // else cout<<"nothing to remove : memory leak ..."<<endl;
543 }catch(std::bad_alloc){
544 cout<<"Warning : Draw option send std::badalloc"<<endl;
553 if (level>checkLevel){
557 focBad<<line<<"\t"<<level<<endl;
562 fic.getline(buffer, sizeof(buffer));
565 //now use chain to check
566 //chain.Lookup(kTRUE);
577 Int_t AliXRDPROOFtoolkit::CheckTreeInFile(const char*fileName,const char*treeName, Int_t debugLevel, const char *branchName){
579 // Check the tree in file
580 // fileName - the name of the file with tree
581 // treeName - the name of file
582 // debugLevel - 0 check the existance of the file - 1 make loop over entries
583 // branchName - if debugLevel>0 the branch is chcecked
584 // if brnachName =0 the content of full tree is chcecked
585 // return value = 0 - Check things OK
586 // -1 - file not exist or not accesible
587 // -2 - file is zombie
588 // -3 - tree not present
589 // -4 - branch not present
590 TFile * file = TFile::Open(fileName);
591 if (!file) { return -1;}
592 if (file->IsZombie()) {file->Close(); delete file; return -2;};
594 TString TrName(treeName);
596 //cout <<" treename ==== *"<<endl;;
597 file->Close(); delete file;
600 TTree * tree = (TTree*)file->Get(treeName);
601 if (!tree) {file->Close(); delete file; return -3;}
602 TBranch * branch = 0;
604 branch = tree->GetBranch(branchName);
605 if (!branch) {file->Close(); delete file; return -4;}
608 tree->SetBranchStatus("*",1);
611 Int_t entries = tree->GetEntries();
612 for (Int_t i=0;i<entries; i++){
613 if (branch) branch->GetEntry(i);
614 else tree->GetEntry();
619 // never catched - as there is no exception in the ROOT IO
620 file->Close(); delete file;
624 file->Close(); delete file;
629 Bool_t AliXRDPROOFtoolkit::FilterList(const char*inputList, const char*fileList, Int_t checkLevel){
632 // inputList - list of original file names
633 // fileList - list of file to be checked
635 // - 1 - treeName (if * not checked)
638 // checkLevel - 0 - check only existance of the files and tree's +
639 // simple file corruption
640 // > 1 - check the content of the tree -
641 // (can crash as there do not exest exception handling in ROOT)
642 // Output - two streams are created - file with good entries
643 // "%s.Good a,d file with bad entries %s.Bad
645 // AliXRDPROOFtoolkit::FilterList("ppgrid2.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",1)
648 finput.open(inputList, ios_base::in);
651 focGood.open(Form("%s.Good",inputList), ios_base::out|ios_base::trunc);
652 focBad.open(Form("%s.Bad",inputList), ios_base::out|ios_base::trunc);
654 if(!finput.is_open()) {
655 cout<<"Can't open file "<<inputList<<endl;
659 // Read the input list of files and add them to the chain
661 TObjArray *array = (TString(fileList)).Tokenize(" ");
664 while(finput.good()) {
665 finput >> currentFile;
666 if (!currentFile.Contains("root")) continue; // protection
667 // Bool_t isZip = currentFile.Contains("#");
668 const char * dirname = gSystem->DirName(currentFile.Data());
670 for (Int_t i=0; i<array->GetEntries(); i+=2){
672 //if (isZip) sprintf(fname,
673 sprintf(fname, "%s/%s",dirname,array->At(i)->GetName());
674 printf("\nFile to be checked %s/%s",dirname,array->At(i)->GetName());
675 //cout <<"\n arguments: "<< array->At(i+1)->GetName()<<" "<<checkLevel<<endl;
676 Int_t cstatus = CheckTreeInFile(fname, array->At(i+1)->GetName(), checkLevel,0);
677 //printf(" CheckTreeInFile returns %d",cstatus);
684 focGood<<currentFile<<endl;
686 focBad<<currentFile<<endl;
697 Bool_t AliXRDPROOFtoolkit::XRDCopyDir(const char * idir, const char * files, const char *odir, Bool_t zip){
699 // idir - input directory
700 // odir - output directory
701 // files - the list of files to be coppied
702 // zip - not supported yet
706 // idir ="root://gsiaf.gsi.de:1094//sma/sim/v4-05-Rev-03/pp/0000";
707 // odir ="root://lxgrid2.gsi.de:1094//miranov/test/pp/0000";
708 // char *files="AliESDs.root AliESDfriend.root Kinematics.root";
710 TObjArray * array = str.Tokenize(" ");
711 Int_t nfiles = array->GetEntries();
715 for (Int_t ifile =0; ifile<nfiles; ifile++){
716 sprintf(infile,"%s/%s", idir, array->At(ifile)->GetName());
717 sprintf(outfile,"%s/%s", odir, array->At(ifile)->GetName());
718 printf("%s - %s\n",infile, outfile);
719 Bool_t result = TFile::Cp(infile,outfile);