--- /dev/null
+/*
+ TTOOLKIT to acces files on XRD
+ WORKING ONLY AT GSI
+//
+
+ Example usage:
+ Load toolkit
+ gSystem->AddIncludePath("-I$ALICE_ROOT/TPC/macros")
+ gROOT->LoadMacro("$ALICE_ROOT/TPC/macros/AliXRDPROOFtoolkit.cxx++")
+
+ 1.
+ Retrieve the list of files : POSSIBLE ONLY FOR USER WITH LOGIN ACCESS TO THE XRD MACHINES
+
+ AliXRDPROOFtoolkit tool;
+ tool.ListOfFiles("pp.txt","/data.local2/sma/sim/v4-05-Rev-03/pp", "AliESDs.root", kTRUE);
+
+
+ 2.
+
+ AliXRDPROOFtoolkit tool;
+ TChain * chain = tool.MakeChain("pp.txt","esdTree",10)
+ chain->Draw("fTPCncls");
+
+
+ 3. Process logs - ONLY priority users - with ssh acces
+ AliXRDPROOFtoolkit tool;
+ tool.FilterSegFault();
+ TTree * treeSys = tool.DumpSys(kTRUE)
+
+*/
+
+
+#include <TTree.h>
+#include <TString.h>
+#include <TObjArray.h>
+#include <TObjString.h>
+#include <TTree.h>
+#include <TFile.h>
+#include <TChain.h>
+#include <TDSet.h>
+#include <TH1.h>
+#include <TGraph.h>
+#include <TMath.h>
+#include <TPad.h>
+#include <exception>
+#include <fstream>
+#include <TRandom.h>
+
+
+#include <AliXRDPROOFtoolkit.h>
+
+
+ClassImp(AliXRDPROOFtoolkit);
+
+
+
+//______________________________________________________________________________
+AliXRDPROOFtoolkit::AliXRDPROOFtoolkit () : TObject ()
+{
+ //
+ // Default -GSI specific setup
+ //
+ for(Int_t i=256;i<268;i++)
+ listeMachine.push_back(new TString(Form("lxb%d.gsi.de", i)));
+ fUserGroup = gSystem->GetUserInfo();
+ fUserName = fUserGroup->fUser;
+ fVerbose=1;
+}
+
+void AliXRDPROOFtoolkit::Print(Option_t* /*option*/) const{
+ //
+ // Print current setup of toolkit
+ //
+ printf("User name\t%s\n", fUserName.Data());
+ printf("List of slaves\n");
+
+ for(UInt_t i=0;i<listeMachine.size();i++)
+ printf("%s\n",listeMachine[i]->Data());
+}
+
+
+//______________________________________________________________________________
+void AliXRDPROOFtoolkit::AddMachine (const char*name)
+{
+ //
+ // add computer to the list
+ //
+ listeMachine.push_back( new TString(name));
+}
+
+
+//______________________________________________________________________________
+Bool_t AliXRDPROOFtoolkit::ListOfFiles(const char*fileName, const char*path, const char*filter, Bool_t displayMachine)
+{
+ //
+ // Get the list of files on "registerd slaves"
+ //
+ // fileName - Resultinfg flie with list
+ // path - starting path on slave e.g /data.local2/sma/
+ // filter - filter expression e.g. AliESDs.root
+ // display machine - not used yet
+
+ char filterXRD[100];
+ char command[1000];
+ //
+ // FILTER expression - to get REDIRECTOR NAME
+ //
+ //what we want for sed
+ // sprintf(filterXRD,"sed s/\\/data.local2/root:\\/\\/gsiaf.gsi.de:1094\\//");
+ //what we should write
+ sprintf(filterXRD,"sed s/\\\\/data.local2/root:\\\\/\\\\/gsiaf.gsi.de:1094\\\\//");
+ //
+ //
+ //
+ gSystem->Exec(Form("echo >%s",fileName));
+ for(UInt_t i=0; i<listeMachine.size(); i++){
+ cout<<"Inspecting "<<listeMachine[i]->Data()<<" ..."<<endl;
+ sprintf(command,"lsrun -m %s find %s | grep %s | %s >> %s", listeMachine[i]->Data(), path, filter, filterXRD,fileName);
+ printf(command);
+ gSystem->Exec(command);
+ }
+ return kTRUE;
+}
+
+
+
+TChain* AliXRDPROOFtoolkit::MakeChain(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles, Int_t startFile)
+{
+ //
+ // Create the chain
+ //
+ TChain* chain = new TChain(treeName);
+
+ // Open the input stream
+ ifstream in;
+ in.open(fileIn);
+
+ // Read the input list of files and add them to the chain
+ TString currentFile;
+ Int_t counter=0;
+ while(in.good()) {
+ in >> currentFile;
+ if (fName) {
+ currentFile+="#";
+ currentFile+=fName;
+ }
+ if (!currentFile.Contains("root")) continue; // protection
+ counter++;
+ if (counter<startFile) continue;
+ if (counter>maxFiles+startFile) break;
+ chain->Add(currentFile.Data());
+ }
+
+ in.close();
+
+ return chain;
+}
+
+TChain* AliXRDPROOFtoolkit::MakeChainRandom(const char*fileIn, const char * treeName,const char *fName, Int_t maxFiles, Int_t startFile)
+{
+ //
+ // Create a TDSet - files are in random order
+ //
+ // filein - input list text file
+ // treename - containg tree
+ // maxFiles - maximum number of files included
+
+ TObjArray array(10000);
+
+ TChain* chain = new TChain(treeName);
+
+ // Open the input stream
+ ifstream in;
+ in.open(fileIn);
+
+ // Read the input list of files and add them to the chain
+ TString currentFile;
+ Int_t counter=0;
+ while(in.good()) {
+ in >> currentFile;
+ if (fName) {
+ currentFile+="#";
+ currentFile+=fName;
+ }
+ if (!currentFile.Contains("root")) continue; // protection
+ counter++;
+ // chain->Add(currentFile.Data());
+ array.AddLast(new TObjString(currentFile));
+ }
+ in.close();
+ Int_t entries = array.GetEntries();
+ printf("Number of entries\t%d",entries);
+ if (maxFiles<0) maxFiles=entries;
+ if (maxFiles>entries) maxFiles=entries;
+ //
+ //
+ //
+ for (Int_t i=0; i<maxFiles; i++){
+ Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
+ if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
+ printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
+ chain->Add(array.At(ifile)->GetName());
+ array.At(ifile)->SetBit(TObject::kCannotPick);
+ }
+ }
+ return chain;
+}
+
+
+
+TDSet* AliXRDPROOFtoolkit::MakeSet(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
+{
+ //
+ // Create the TDSet out of list
+ // filein - input list text file
+ // treename - containg tree
+ // maxFiles - maximum number of files included
+
+ TDSet* chain = new TDSet(treeName);
+
+ // Open the input stream
+ ifstream in;
+ in.open(fileIn);
+
+ // Read the input list of files and add them to the chain
+ TString currentFile;
+ Int_t counter=0;
+ while(in.good()) {
+ in >> currentFile;
+ if (fName) {
+ currentFile+="#";
+ currentFile+=fName;
+ }
+ if (!currentFile.Contains("root")) continue; // protection
+ counter++;
+ if (maxFiles>0 && counter>maxFiles) break;
+ chain->Add(currentFile.Data());
+ }
+
+ in.close();
+ chain->Validate();
+ return chain;
+}
+
+
+TDSet* AliXRDPROOFtoolkit::MakeSetRandom(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
+{
+ //
+ // Create a TDSet - files are in random order
+ //
+ // filein - input list text file
+ // treename - containg tree
+ // maxFiles - maximum number of files included
+
+ TObjArray array(10000);
+
+ TDSet* chain = new TDSet(treeName);
+
+ // Open the input stream
+ ifstream in;
+ in.open(fileIn);
+
+ // Read the input list of files and add them to the chain
+ TString currentFile;
+ Int_t counter=0;
+ while(in.good()) {
+ in >> currentFile;
+ if (fName) {
+ currentFile+="#";
+ currentFile+=fName;
+ }
+ if (!currentFile.Contains("root")) continue; // protection
+ counter++;
+ // chain->Add(currentFile.Data());
+ array.AddLast(new TObjString(currentFile));
+ }
+ in.close();
+ Int_t entries = array.GetEntries();
+ printf("Number of entries\t%d",entries);
+ if (maxFiles<0) maxFiles=entries;
+ if (maxFiles>entries) maxFiles=entries;
+ for (Int_t i=0; i<maxFiles; i++){
+ Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
+ if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
+ printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
+ chain->Add(array.At(ifile)->GetName());
+ array.At(ifile)->SetBit(TObject::kCannotPick);
+ }
+ }
+
+
+ chain->Validate();
+ return chain;
+}
+
+//_______________________________________________________________________________
+
+
+void AliXRDPROOFtoolkit::FilterSegFault(const char *filter){
+ //
+ // Print the list of slaves finished with seg fault
+ //
+ printf("List of the PROOF slaves with seg fault (Last session) \n");
+ for(UInt_t i=0; i<listeMachine.size(); i++){
+ if (HasSegFault(listeMachine[i]->Data(), filter)){
+ printf("%s\n",listeMachine[i]->Data());
+ }
+ }
+}
+
+//_______________________________________________________________________________
+
+Bool_t AliXRDPROOFtoolkit::HasSegFault(const char * machine, const char *filter){
+ //
+ // check if segmentation fault on the node
+ //
+ char command0[1000];
+ char command[1000];
+ char *path = "segfault.log";
+ Long_t id,size,flags,modtime;
+
+ // first check if we have a log file
+ sprintf(command0, Form("lsrun -m %s find /data.local2/proof/%s/ | grep %s | grep .log >filelist.list", machine,fUserName.Data(), filter));
+ if (fVerbose>1) {
+ printf("%s\n",command0);
+ }
+ gSystem->Exec(command0);
+ gSystem->GetPathInfo("filelist.list", &id, &size, &flags, &modtime);
+ if (size<1) return kFALSE;
+
+ //
+ // now check the content
+ //
+ sprintf(command, Form("lsrun -m %s cat `lsrun -m %s find /data.local2/proof/%s/ | grep %s|grep log ` | grep segmentation >segfault.log", machine, machine,fUserName.Data(), filter));
+
+ if (fVerbose) {
+ printf("%s\n",command);
+ }
+ if (fVerbose>1) gSystem->Exec(command);
+ gSystem->GetPathInfo(path, &id, &size, &flags, &modtime);
+ return (size>10);
+}
+
+
+//--------------------------------------------------------------------
+
+TTree * AliXRDPROOFtoolkit::DumpSys(Bool_t verbose){
+ //
+ // Dump memory usage on worker nodes
+ gSystem->Exec("echo > syswatch.log");
+ for(UInt_t i=0; i<listeMachine.size(); i++){
+ if (verbose) gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data()));
+ gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log >> syswatch.log",listeMachine[i]->Data()));
+ }
+
+ TTree * tree = new TTree("asci","asci");
+ tree->SetDirectory(0);
+ // tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F");
+ tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F");
+ return tree;
+}
+
+// //--------------------------------------------------------------------
+
+TTree * AliXRDPROOFtoolkit::DumpSys2(Bool_t verbose){
+ //
+ // Dump memory usage on worker nodes
+ //
+ gSystem->Exec("echo > syswatch.log");
+ for(UInt_t i=0; i<listeMachine.size(); i++){
+ if (verbose)
+ gSystem->Exec(Form("lsrun -m %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data()));
+ gSystem->Exec(Form("lsrun -m %s cat /data.local2/proof/miranov/last-workersession/syswatch.log >> syswatch.log",listeMachine[i]->Data()));
+ }
+
+ TTree * tree = new TTree("asci","asci");
+ tree->SetDirectory(0);
+ // tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F");
+ tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F");
+ return tree;
+}
+
+
+
+
+//
+
+
+TTree* AliXRDPROOFtoolkit::DumpFiles(Bool_t verbose){
+ //
+ //
+ //
+ gSystem->Exec("touch filewatch.log");
+ for(UInt_t i=0; i<listeMachine.size(); i++){
+ if (verbose) gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/worker*/filewatch.log >>filewatch.log",listeMachine[i]->Data()));
+ gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/worker*/filewatch.log >>filewatch.log",listeMachine[i]->Data()));
+ }
+ TTree * tree = new TTree("asci","asci");
+ tree->SetDirectory(0);
+ tree->ReadFile("filewatch.log","hname/C:fname/C:entry/I:status/I");
+ return tree;
+}
+
+
+
+
+
+//______________________________________________________________________________
+Int_t AliXRDPROOFtoolkit::Read(char * str, Int_t lenght, FILE *in)
+{
+ Int_t nread=0;
+ Int_t n;
+
+ while (lenght>0) {
+ n=fread(str+nread, sizeof(char), lenght, in);
+ if(n<=0) return nread;
+ nread+=n;
+ lenght-=n;
+ }
+ return nread;
+}
+
+
+
+
+//______________________________________________________________________________
+void AliXRDPROOFtoolkit::CheckFiles (const char*fileIn, UInt_t checkLevel, const char*treeToRetrieve, const char*varexp, const char*selection)
+{
+ //
+ // check the files
+ //
+ fstream fic;
+ fic.open(fileIn, ios_base::in);
+ fstream focGood;
+ fstream focBad;
+ focGood.open(Form("%s.Good",fileIn), ios_base::out|ios_base::trunc);
+ focBad.open(Form("%s.Bad",fileIn), ios_base::out|ios_base::trunc);
+ //
+ if(!fic.is_open()) {
+ cout<<"Can't open file "<<fileIn<<endl;
+ return;
+ }
+ //
+ //
+ //
+ Long64_t size;
+ char buffer[256];
+ char * line;
+ char * machine;
+ Int_t level=0;
+ Float_t compressionFactor;
+ Int_t nkey;
+ Int_t version;
+ TObjString * fileName=0x0;
+ TObjString * machineName=0x0;
+ // TChain chain ("check_chain");
+
+ TFile fout(Form("%s.root",fileIn),"recreate");
+ TTree * tree=new TTree("stats", "stats of AliXRDPROOFtoolkit::CheckFiles function");
+
+ tree->Branch ("machine", "TObjString", &machineName, 256000,0);
+ tree->Branch ("file", "TObjString", &fileName, 256000,0);
+ tree->Branch ("level", &level, "level/I");
+ tree->Branch ("size", &size, "size/L");
+ tree->Branch ("nkey", &nkey, "nkey/I");
+ tree->Branch ("compress", &compressionFactor, "compress/F");
+ tree->Branch ("version", &version, "version/I");
+
+ // start loop over the files
+ fic.getline(buffer, sizeof(buffer));
+ while (fic.good()){
+
+ // get the machine name if present in the file
+ machine=strchr(buffer, '\t');
+ if(machine!=0x0){
+ machine[0]='\0';
+ line=machine+1;
+ machine=buffer;
+ machineName=new TObjString(machine);
+ }else {
+ machineName=new TObjString("x");
+ line=buffer;
+ }
+ cout<<"Inspecting file :"<<line<<endl;
+
+
+ TTree * getTree=0x0;
+ fileName=new TObjString(line);
+
+ TFile * f=TFile::Open(line);
+ // chain.AddFile(line);
+ level=0;
+ size=-1;
+ nkey=-1;
+ compressionFactor=-1;
+ version=-1;
+
+ if (fileName->String().Contains("AliESDs.root")){
+ //
+ // check friend file
+ //
+ char fnameFriend[1000];
+ sprintf(fnameFriend,"%s/AliESDfriends.root",gSystem->DirName(fileName->String().Data()));
+ cout<<"Inspecting file :"<<fnameFriend<<endl;
+ TFile * ffriend=TFile::Open(fnameFriend);
+ if (!ffriend) level=-4;
+ else{
+ if (ffriend->IsZombie()) level=-3;
+ if (ffriend->TestBit(TFile::kRecovered) || ffriend->TestBit(TFile::kWriteError)){
+ level=-2;
+ }
+ ffriend->Close();
+ delete ffriend;
+ }
+ }
+
+
+ if(level>=-1 && f!=0x0){
+ level=1;
+ size=f->GetSize();
+ nkey=f->GetNkeys();
+ compressionFactor=f->GetCompressionFactor();
+ version=f->GetVersion();
+
+ if(checkLevel>0 && !f->IsZombie()){
+ level=2;
+ if(checkLevel>1 && treeToRetrieve!="" && (getTree=(TTree*)f->Get(treeToRetrieve))!=0x0){
+ level=3;
+ Int_t tentries = getTree->GetEntries();
+ if (tentries>=0) level=4;
+ cout<<"Number of entries :"<<getTree->GetEntries()<<endl;
+
+ if(checkLevel>3 && varexp!="" &&tentries>0) {
+ getTree->SetBranchStatus("*",1);
+ try{
+ TH1F his("his","his",100,-1,1);
+ Int_t selected = getTree->Draw(Form("%s>>his",varexp), selection, "goff", 1, tentries-1);
+ cout<<"Selected\t"<<selected<<endl;
+ if(selected>-1){
+ level=5;
+
+ //try to remove the created histogrames ...
+// TH1F *htemp = (TH1F*)gPad->GetPrimitive("htemp"); // 1D
+// TGraph *graph = (TGraph*)gPad->GetPrimitive("Graph"); // 2D
+// if(htemp!=0x0) {cout<<"removing TH1D"<<endl; delete htemp;}
+// else if(graph!=0x0) {cout<<"removing TGraph"<<endl; delete graph;}
+// else cout<<"nothing to remove : memory leak ..."<<endl;
+
+ }
+ }catch(std::bad_alloc){
+ cout<<"Warning : Draw option send std::badalloc"<<endl;
+ }
+ }
+ delete getTree;
+ }
+ }
+ f->Close();
+ delete f;
+ }
+ if (level>checkLevel){
+ focGood<<line<<endl;
+ }
+ else{
+ focBad<<line<<"\t"<<level<<endl;
+ }
+
+
+ tree->Fill();
+ fic.getline(buffer, sizeof(buffer));
+ }
+
+ //now use chain to check
+ //chain.Lookup(kTRUE);
+
+
+ // Save the tree
+ tree->Write();
+ fout.Close();
+ focGood.close();
+ focBad.close();
+
+}
+
+
+
+
+
git://git.uio.no / u / mrichter / AliRoot.git / commitdiff