From: marian Date: Tue, 23 Oct 2007 12:22:11 +0000 (+0000) Subject: Ading toolkit for accesing files on XRD (Marian) X-Git-Url: http://git.uio.no/git/?a=commitdiff_plain;h=db6fdd20618705a34767d258bb4bbb9ebaa3fd60;p=u%2Fmrichter%2FAliRoot.git Ading toolkit for accesing files on XRD (Marian) --- diff --git a/TPC/macros/AliXRDPROOFtoolkit.cxx b/TPC/macros/AliXRDPROOFtoolkit.cxx new file mode 100644 index 00000000000..6b04d07a228 --- /dev/null +++ b/TPC/macros/AliXRDPROOFtoolkit.cxx @@ -0,0 +1,587 @@ +/* + TTOOLKIT to acces files on XRD + WORKING ONLY AT GSI +// + + Example usage: + Load toolkit + gSystem->AddIncludePath("-I$ALICE_ROOT/TPC/macros") + gROOT->LoadMacro("$ALICE_ROOT/TPC/macros/AliXRDPROOFtoolkit.cxx++") + + 1. + Retrieve the list of files : POSSIBLE ONLY FOR USER WITH LOGIN ACCESS TO THE XRD MACHINES + + AliXRDPROOFtoolkit tool; + tool.ListOfFiles("pp.txt","/data.local2/sma/sim/v4-05-Rev-03/pp", "AliESDs.root", kTRUE); + + + 2. + + AliXRDPROOFtoolkit tool; + TChain * chain = tool.MakeChain("pp.txt","esdTree",10) + chain->Draw("fTPCncls"); + + + 3. Process logs - ONLY priority users - with ssh acces + AliXRDPROOFtoolkit tool; + tool.FilterSegFault(); + TTree * treeSys = tool.DumpSys(kTRUE) + +*/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include + + +ClassImp(AliXRDPROOFtoolkit); + + + +//______________________________________________________________________________ +AliXRDPROOFtoolkit::AliXRDPROOFtoolkit () : TObject () +{ + // + // Default -GSI specific setup + // + for(Int_t i=256;i<268;i++) + listeMachine.push_back(new TString(Form("lxb%d.gsi.de", i))); + fUserGroup = gSystem->GetUserInfo(); + fUserName = fUserGroup->fUser; + fVerbose=1; +} + +void AliXRDPROOFtoolkit::Print(Option_t* /*option*/) const{ + // + // Print current setup of toolkit + // + printf("User name\t%s\n", fUserName.Data()); + printf("List of slaves\n"); + + for(UInt_t i=0;iData()); +} + + +//______________________________________________________________________________ +void AliXRDPROOFtoolkit::AddMachine (const char*name) +{ + // + // add computer to the list + // + listeMachine.push_back( new TString(name)); +} + + +//______________________________________________________________________________ +Bool_t AliXRDPROOFtoolkit::ListOfFiles(const char*fileName, const char*path, const char*filter, Bool_t displayMachine) +{ + // + // Get the list of files on "registerd slaves" + // + // fileName - Resultinfg flie with list + // path - starting path on slave e.g /data.local2/sma/ + // filter - filter expression e.g. AliESDs.root + // display machine - not used yet + + char filterXRD[100]; + char command[1000]; + // + // FILTER expression - to get REDIRECTOR NAME + // + //what we want for sed + // sprintf(filterXRD,"sed s/\\/data.local2/root:\\/\\/gsiaf.gsi.de:1094\\//"); + //what we should write + sprintf(filterXRD,"sed s/\\\\/data.local2/root:\\\\/\\\\/gsiaf.gsi.de:1094\\\\//"); + // + // + // + gSystem->Exec(Form("echo >%s",fileName)); + for(UInt_t i=0; iData()<<" ..."<> %s", listeMachine[i]->Data(), path, filter, filterXRD,fileName); + printf(command); + gSystem->Exec(command); + } + return kTRUE; +} + + + +TChain* AliXRDPROOFtoolkit::MakeChain(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles, Int_t startFile) +{ + // + // Create the chain + // + TChain* chain = new TChain(treeName); + + // Open the input stream + ifstream in; + in.open(fileIn); + + // Read the input list of files and add them to the chain + TString currentFile; + Int_t counter=0; + while(in.good()) { + in >> currentFile; + if (fName) { + currentFile+="#"; + currentFile+=fName; + } + if (!currentFile.Contains("root")) continue; // protection + counter++; + if (countermaxFiles+startFile) break; + chain->Add(currentFile.Data()); + } + + in.close(); + + return chain; +} + +TChain* AliXRDPROOFtoolkit::MakeChainRandom(const char*fileIn, const char * treeName,const char *fName, Int_t maxFiles, Int_t startFile) +{ + // + // Create a TDSet - files are in random order + // + // filein - input list text file + // treename - containg tree + // maxFiles - maximum number of files included + + TObjArray array(10000); + + TChain* chain = new TChain(treeName); + + // Open the input stream + ifstream in; + in.open(fileIn); + + // Read the input list of files and add them to the chain + TString currentFile; + Int_t counter=0; + while(in.good()) { + in >> currentFile; + if (fName) { + currentFile+="#"; + currentFile+=fName; + } + if (!currentFile.Contains("root")) continue; // protection + counter++; + // chain->Add(currentFile.Data()); + array.AddLast(new TObjString(currentFile)); + } + in.close(); + Int_t entries = array.GetEntries(); + printf("Number of entries\t%d",entries); + if (maxFiles<0) maxFiles=entries; + if (maxFiles>entries) maxFiles=entries; + // + // + // + for (Int_t i=0; iRndm()*Float_t(entries)); + if (ifileTestBit(TObject::kCannotPick)==kFALSE){ + printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName()); + chain->Add(array.At(ifile)->GetName()); + array.At(ifile)->SetBit(TObject::kCannotPick); + } + } + return chain; +} + + + +TDSet* AliXRDPROOFtoolkit::MakeSet(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles) +{ + // + // Create the TDSet out of list + // filein - input list text file + // treename - containg tree + // maxFiles - maximum number of files included + + TDSet* chain = new TDSet(treeName); + + // Open the input stream + ifstream in; + in.open(fileIn); + + // Read the input list of files and add them to the chain + TString currentFile; + Int_t counter=0; + while(in.good()) { + in >> currentFile; + if (fName) { + currentFile+="#"; + currentFile+=fName; + } + if (!currentFile.Contains("root")) continue; // protection + counter++; + if (maxFiles>0 && counter>maxFiles) break; + chain->Add(currentFile.Data()); + } + + in.close(); + chain->Validate(); + return chain; +} + + +TDSet* AliXRDPROOFtoolkit::MakeSetRandom(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles) +{ + // + // Create a TDSet - files are in random order + // + // filein - input list text file + // treename - containg tree + // maxFiles - maximum number of files included + + TObjArray array(10000); + + TDSet* chain = new TDSet(treeName); + + // Open the input stream + ifstream in; + in.open(fileIn); + + // Read the input list of files and add them to the chain + TString currentFile; + Int_t counter=0; + while(in.good()) { + in >> currentFile; + if (fName) { + currentFile+="#"; + currentFile+=fName; + } + if (!currentFile.Contains("root")) continue; // protection + counter++; + // chain->Add(currentFile.Data()); + array.AddLast(new TObjString(currentFile)); + } + in.close(); + Int_t entries = array.GetEntries(); + printf("Number of entries\t%d",entries); + if (maxFiles<0) maxFiles=entries; + if (maxFiles>entries) maxFiles=entries; + for (Int_t i=0; iRndm()*Float_t(entries)); + if (ifileTestBit(TObject::kCannotPick)==kFALSE){ + printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName()); + chain->Add(array.At(ifile)->GetName()); + array.At(ifile)->SetBit(TObject::kCannotPick); + } + } + + + chain->Validate(); + return chain; +} + +//_______________________________________________________________________________ + + +void AliXRDPROOFtoolkit::FilterSegFault(const char *filter){ + // + // Print the list of slaves finished with seg fault + // + printf("List of the PROOF slaves with seg fault (Last session) \n"); + for(UInt_t i=0; iData(), filter)){ + printf("%s\n",listeMachine[i]->Data()); + } + } +} + +//_______________________________________________________________________________ + +Bool_t AliXRDPROOFtoolkit::HasSegFault(const char * machine, const char *filter){ + // + // check if segmentation fault on the node + // + char command0[1000]; + char command[1000]; + char *path = "segfault.log"; + Long_t id,size,flags,modtime; + + // first check if we have a log file + sprintf(command0, Form("lsrun -m %s find /data.local2/proof/%s/ | grep %s | grep .log >filelist.list", machine,fUserName.Data(), filter)); + if (fVerbose>1) { + printf("%s\n",command0); + } + gSystem->Exec(command0); + gSystem->GetPathInfo("filelist.list", &id, &size, &flags, &modtime); + if (size<1) return kFALSE; + + // + // now check the content + // + sprintf(command, Form("lsrun -m %s cat `lsrun -m %s find /data.local2/proof/%s/ | grep %s|grep log ` | grep segmentation >segfault.log", machine, machine,fUserName.Data(), filter)); + + if (fVerbose) { + printf("%s\n",command); + } + if (fVerbose>1) gSystem->Exec(command); + gSystem->GetPathInfo(path, &id, &size, &flags, &modtime); + return (size>10); +} + + +//-------------------------------------------------------------------- + +TTree * AliXRDPROOFtoolkit::DumpSys(Bool_t verbose){ + // + // Dump memory usage on worker nodes + gSystem->Exec("echo > syswatch.log"); + for(UInt_t i=0; iExec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data())); + gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log >> syswatch.log",listeMachine[i]->Data())); + } + + TTree * tree = new TTree("asci","asci"); + tree->SetDirectory(0); + // tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F"); + tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F"); + return tree; +} + +// //-------------------------------------------------------------------- + +TTree * AliXRDPROOFtoolkit::DumpSys2(Bool_t verbose){ + // + // Dump memory usage on worker nodes + // + gSystem->Exec("echo > syswatch.log"); + for(UInt_t i=0; iExec(Form("lsrun -m %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data())); + gSystem->Exec(Form("lsrun -m %s cat /data.local2/proof/miranov/last-workersession/syswatch.log >> syswatch.log",listeMachine[i]->Data())); + } + + TTree * tree = new TTree("asci","asci"); + tree->SetDirectory(0); + // tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F"); + tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F"); + return tree; +} + + + + +// + + +TTree* AliXRDPROOFtoolkit::DumpFiles(Bool_t verbose){ + // + // + // + gSystem->Exec("touch filewatch.log"); + for(UInt_t i=0; iExec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/worker*/filewatch.log >>filewatch.log",listeMachine[i]->Data())); + gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/worker*/filewatch.log >>filewatch.log",listeMachine[i]->Data())); + } + TTree * tree = new TTree("asci","asci"); + tree->SetDirectory(0); + tree->ReadFile("filewatch.log","hname/C:fname/C:entry/I:status/I"); + return tree; +} + + + + + +//______________________________________________________________________________ +Int_t AliXRDPROOFtoolkit::Read(char * str, Int_t lenght, FILE *in) +{ + Int_t nread=0; + Int_t n; + + while (lenght>0) { + n=fread(str+nread, sizeof(char), lenght, in); + if(n<=0) return nread; + nread+=n; + lenght-=n; + } + return nread; +} + + + + +//______________________________________________________________________________ +void AliXRDPROOFtoolkit::CheckFiles (const char*fileIn, UInt_t checkLevel, const char*treeToRetrieve, const char*varexp, const char*selection) +{ + // + // check the files + // + fstream fic; + fic.open(fileIn, ios_base::in); + fstream focGood; + fstream focBad; + focGood.open(Form("%s.Good",fileIn), ios_base::out|ios_base::trunc); + focBad.open(Form("%s.Bad",fileIn), ios_base::out|ios_base::trunc); + // + if(!fic.is_open()) { + cout<<"Can't open file "<Branch ("machine", "TObjString", &machineName, 256000,0); + tree->Branch ("file", "TObjString", &fileName, 256000,0); + tree->Branch ("level", &level, "level/I"); + tree->Branch ("size", &size, "size/L"); + tree->Branch ("nkey", &nkey, "nkey/I"); + tree->Branch ("compress", &compressionFactor, "compress/F"); + tree->Branch ("version", &version, "version/I"); + + // start loop over the files + fic.getline(buffer, sizeof(buffer)); + while (fic.good()){ + + // get the machine name if present in the file + machine=strchr(buffer, '\t'); + if(machine!=0x0){ + machine[0]='\0'; + line=machine+1; + machine=buffer; + machineName=new TObjString(machine); + }else { + machineName=new TObjString("x"); + line=buffer; + } + cout<<"Inspecting file :"<String().Contains("AliESDs.root")){ + // + // check friend file + // + char fnameFriend[1000]; + sprintf(fnameFriend,"%s/AliESDfriends.root",gSystem->DirName(fileName->String().Data())); + cout<<"Inspecting file :"<IsZombie()) level=-3; + if (ffriend->TestBit(TFile::kRecovered) || ffriend->TestBit(TFile::kWriteError)){ + level=-2; + } + ffriend->Close(); + delete ffriend; + } + } + + + if(level>=-1 && f!=0x0){ + level=1; + size=f->GetSize(); + nkey=f->GetNkeys(); + compressionFactor=f->GetCompressionFactor(); + version=f->GetVersion(); + + if(checkLevel>0 && !f->IsZombie()){ + level=2; + if(checkLevel>1 && treeToRetrieve!="" && (getTree=(TTree*)f->Get(treeToRetrieve))!=0x0){ + level=3; + Int_t tentries = getTree->GetEntries(); + if (tentries>=0) level=4; + cout<<"Number of entries :"<GetEntries()<3 && varexp!="" &&tentries>0) { + getTree->SetBranchStatus("*",1); + try{ + TH1F his("his","his",100,-1,1); + Int_t selected = getTree->Draw(Form("%s>>his",varexp), selection, "goff", 1, tentries-1); + cout<<"Selected\t"<-1){ + level=5; + + //try to remove the created histogrames ... +// TH1F *htemp = (TH1F*)gPad->GetPrimitive("htemp"); // 1D +// TGraph *graph = (TGraph*)gPad->GetPrimitive("Graph"); // 2D +// if(htemp!=0x0) {cout<<"removing TH1D"<Close(); + delete f; + } + if (level>checkLevel){ + focGood<Fill(); + fic.getline(buffer, sizeof(buffer)); + } + + //now use chain to check + //chain.Lookup(kTRUE); + + + // Save the tree + tree->Write(); + fout.Close(); + focGood.close(); + focBad.close(); + +} + + + + + diff --git a/TPC/macros/AliXRDPROOFtoolkit.h b/TPC/macros/AliXRDPROOFtoolkit.h new file mode 100644 index 00000000000..90b79a2ed68 --- /dev/null +++ b/TPC/macros/AliXRDPROOFtoolkit.h @@ -0,0 +1,62 @@ +#ifndef _AliXRDPROOFtoolkit +#define _AliXRDPROOFtoolkit + +#include +#include + +#include +#include +#include + +using namespace std; + +class TObjArray; +class TTree; +class TChain; +class TDSet; + + + +class AliXRDPROOFtoolkit : public TObject +{ + + + public : + AliXRDPROOFtoolkit (); + void Print(Option_t* option = " ") const; + public : + // + // Interface for low priority users - NO ssh ACCESS to PROOF machines + // + TChain * MakeChain(const char*fileIn, const char * treeName, const char *fName=0, Int_t maxFiles=-1, Int_t startFile=0); + TChain * MakeChainRandom(const char*fileIn, const char * treeName, const char *fName=0, Int_t maxFiles=-1, Int_t startFile=0); + TDSet * MakeSet(const char*fileIn, const char * treeName, const char *fName=0, Int_t maxFiles=-1); + TDSet * MakeSetRandom(const char*fileIn, const char * treeName,const char *fName=0, Int_t maxFiles=-1); + // + // Interface for users with privileges - Possible to use lsrun command + // + Bool_t ListOfFiles(const char*fileName, const char*path, const char*filter, Bool_t displayMachine); + // + // Interface for users with ssh access to the machines + // + void FilterSegFault(const char *filter="last"); + Bool_t HasSegFault(const char * machine, const char * filter = "last"); + TTree * DumpSys(Bool_t verbose=kTRUE); + TTree * DumpSys2(Bool_t verbose=kTRUE); + TTree * DumpFiles(Bool_t verbose=kTRUE); + // + void CheckFiles (const char*fileIn, UInt_t checkLevel, const char*treeToRetrieve, const char*varexp, const char*selection); + void AddMachine (const char*name); + Int_t fVerbose; // verbso mode - print command + private : + // + // + // + Int_t Read(char * str, Int_t lenght, FILE *in); + private: + vector listeMachine; // list of slaves + TString fUserName; // user name + UserGroup_t *fUserGroup; // user group info + ClassDef(AliXRDPROOFtoolkit, 0); +}; +#endif