correction in usage example
[u/mrichter/AliRoot.git] / TPC / macros / AliXRDPROOFtoolkit.cxx
CommitLineData
db6fdd20 1/*
2 TTOOLKIT to acces files on XRD
3 WORKING ONLY AT GSI
4//
5
6 Example usage:
7 Load toolkit
8 gSystem->AddIncludePath("-I$ALICE_ROOT/TPC/macros")
9 gROOT->LoadMacro("$ALICE_ROOT/TPC/macros/AliXRDPROOFtoolkit.cxx++")
10
11 1.
12 Retrieve the list of files : POSSIBLE ONLY FOR USER WITH LOGIN ACCESS TO THE XRD MACHINES
13
14 AliXRDPROOFtoolkit tool;
15 tool.ListOfFiles("pp.txt","/data.local2/sma/sim/v4-05-Rev-03/pp", "AliESDs.root", kTRUE);
16
17
18 2.
19
20 AliXRDPROOFtoolkit tool;
21 TChain * chain = tool.MakeChain("pp.txt","esdTree",10)
22 chain->Draw("fTPCncls");
23
24
25 3. Process logs - ONLY priority users - with ssh acces
26 AliXRDPROOFtoolkit tool;
27 tool.FilterSegFault();
28 TTree * treeSys = tool.DumpSys(kTRUE)
29
30*/
31
32
33#include <TTree.h>
34#include <TString.h>
35#include <TObjArray.h>
36#include <TObjString.h>
37#include <TTree.h>
38#include <TFile.h>
39#include <TChain.h>
40#include <TDSet.h>
41#include <TH1.h>
42#include <TGraph.h>
43#include <TMath.h>
44#include <TPad.h>
45#include <exception>
46#include <fstream>
47#include <TRandom.h>
48
49
50#include <AliXRDPROOFtoolkit.h>
51
52
53ClassImp(AliXRDPROOFtoolkit);
54
55
56
57//______________________________________________________________________________
58AliXRDPROOFtoolkit::AliXRDPROOFtoolkit () : TObject ()
59{
60 //
61 // Default -GSI specific setup
62 //
63 for(Int_t i=256;i<268;i++)
64 listeMachine.push_back(new TString(Form("lxb%d.gsi.de", i)));
65 fUserGroup = gSystem->GetUserInfo();
66 fUserName = fUserGroup->fUser;
67 fVerbose=1;
68}
69
70void AliXRDPROOFtoolkit::Print(Option_t* /*option*/) const{
71 //
72 // Print current setup of toolkit
73 //
74 printf("User name\t%s\n", fUserName.Data());
75 printf("List of slaves\n");
76
77 for(UInt_t i=0;i<listeMachine.size();i++)
78 printf("%s\n",listeMachine[i]->Data());
79}
80
81
82//______________________________________________________________________________
83void AliXRDPROOFtoolkit::AddMachine (const char*name)
84{
85 //
86 // add computer to the list
87 //
88 listeMachine.push_back( new TString(name));
89}
90
91
92//______________________________________________________________________________
93Bool_t AliXRDPROOFtoolkit::ListOfFiles(const char*fileName, const char*path, const char*filter, Bool_t displayMachine)
94{
95 //
96 // Get the list of files on "registerd slaves"
97 //
98 // fileName - Resultinfg flie with list
99 // path - starting path on slave e.g /data.local2/sma/
100 // filter - filter expression e.g. AliESDs.root
101 // display machine - not used yet
102
103 char filterXRD[100];
104 char command[1000];
105 //
106 // FILTER expression - to get REDIRECTOR NAME
107 //
108 //what we want for sed
109 // sprintf(filterXRD,"sed s/\\/data.local2/root:\\/\\/gsiaf.gsi.de:1094\\//");
110 //what we should write
111 sprintf(filterXRD,"sed s/\\\\/data.local2/root:\\\\/\\\\/gsiaf.gsi.de:1094\\\\//");
112 //
113 //
114 //
115 gSystem->Exec(Form("echo >%s",fileName));
116 for(UInt_t i=0; i<listeMachine.size(); i++){
117 cout<<"Inspecting "<<listeMachine[i]->Data()<<" ..."<<endl;
118 sprintf(command,"lsrun -m %s find %s | grep %s | %s >> %s", listeMachine[i]->Data(), path, filter, filterXRD,fileName);
119 printf(command);
120 gSystem->Exec(command);
121 }
122 return kTRUE;
123}
124
125
126
127TChain* AliXRDPROOFtoolkit::MakeChain(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles, Int_t startFile)
128{
129 //
130 // Create the chain
131 //
132 TChain* chain = new TChain(treeName);
133
134 // Open the input stream
135 ifstream in;
136 in.open(fileIn);
137
138 // Read the input list of files and add them to the chain
139 TString currentFile;
140 Int_t counter=0;
141 while(in.good()) {
142 in >> currentFile;
143 if (fName) {
144 currentFile+="#";
145 currentFile+=fName;
146 }
147 if (!currentFile.Contains("root")) continue; // protection
148 counter++;
149 if (counter<startFile) continue;
150 if (counter>maxFiles+startFile) break;
151 chain->Add(currentFile.Data());
152 }
153
154 in.close();
155
156 return chain;
157}
158
159TChain* AliXRDPROOFtoolkit::MakeChainRandom(const char*fileIn, const char * treeName,const char *fName, Int_t maxFiles, Int_t startFile)
160{
161 //
162 // Create a TDSet - files are in random order
163 //
164 // filein - input list text file
165 // treename - containg tree
166 // maxFiles - maximum number of files included
167
168 TObjArray array(10000);
169
170 TChain* chain = new TChain(treeName);
171
172 // Open the input stream
173 ifstream in;
174 in.open(fileIn);
175
176 // Read the input list of files and add them to the chain
177 TString currentFile;
178 Int_t counter=0;
179 while(in.good()) {
180 in >> currentFile;
181 if (fName) {
182 currentFile+="#";
183 currentFile+=fName;
184 }
185 if (!currentFile.Contains("root")) continue; // protection
186 counter++;
187 // chain->Add(currentFile.Data());
188 array.AddLast(new TObjString(currentFile));
189 }
190 in.close();
191 Int_t entries = array.GetEntries();
192 printf("Number of entries\t%d",entries);
193 if (maxFiles<0) maxFiles=entries;
194 if (maxFiles>entries) maxFiles=entries;
195 //
196 //
197 //
198 for (Int_t i=0; i<maxFiles; i++){
199 Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
200 if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
201 printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
202 chain->Add(array.At(ifile)->GetName());
203 array.At(ifile)->SetBit(TObject::kCannotPick);
204 }
205 }
206 return chain;
207}
208
209
210
211TDSet* AliXRDPROOFtoolkit::MakeSet(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
212{
213 //
214 // Create the TDSet out of list
215 // filein - input list text file
216 // treename - containg tree
217 // maxFiles - maximum number of files included
218
219 TDSet* chain = new TDSet(treeName);
220
221 // Open the input stream
222 ifstream in;
223 in.open(fileIn);
224
225 // Read the input list of files and add them to the chain
226 TString currentFile;
227 Int_t counter=0;
228 while(in.good()) {
229 in >> currentFile;
230 if (fName) {
231 currentFile+="#";
232 currentFile+=fName;
233 }
234 if (!currentFile.Contains("root")) continue; // protection
235 counter++;
236 if (maxFiles>0 && counter>maxFiles) break;
237 chain->Add(currentFile.Data());
238 }
239
240 in.close();
241 chain->Validate();
242 return chain;
243}
244
245
246TDSet* AliXRDPROOFtoolkit::MakeSetRandom(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
247{
248 //
249 // Create a TDSet - files are in random order
250 //
251 // filein - input list text file
252 // treename - containg tree
253 // maxFiles - maximum number of files included
254
255 TObjArray array(10000);
256
257 TDSet* chain = new TDSet(treeName);
258
259 // Open the input stream
260 ifstream in;
261 in.open(fileIn);
262
263 // Read the input list of files and add them to the chain
264 TString currentFile;
265 Int_t counter=0;
266 while(in.good()) {
267 in >> currentFile;
268 if (fName) {
269 currentFile+="#";
270 currentFile+=fName;
271 }
272 if (!currentFile.Contains("root")) continue; // protection
273 counter++;
274 // chain->Add(currentFile.Data());
275 array.AddLast(new TObjString(currentFile));
276 }
277 in.close();
278 Int_t entries = array.GetEntries();
279 printf("Number of entries\t%d",entries);
280 if (maxFiles<0) maxFiles=entries;
281 if (maxFiles>entries) maxFiles=entries;
282 for (Int_t i=0; i<maxFiles; i++){
283 Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
284 if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
285 printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
286 chain->Add(array.At(ifile)->GetName());
287 array.At(ifile)->SetBit(TObject::kCannotPick);
288 }
289 }
290
291
292 chain->Validate();
293 return chain;
294}
295
296//_______________________________________________________________________________
297
298
299void AliXRDPROOFtoolkit::FilterSegFault(const char *filter){
300 //
301 // Print the list of slaves finished with seg fault
302 //
303 printf("List of the PROOF slaves with seg fault (Last session) \n");
304 for(UInt_t i=0; i<listeMachine.size(); i++){
305 if (HasSegFault(listeMachine[i]->Data(), filter)){
306 printf("%s\n",listeMachine[i]->Data());
307 }
308 }
309}
310
311//_______________________________________________________________________________
312
313Bool_t AliXRDPROOFtoolkit::HasSegFault(const char * machine, const char *filter){
314 //
315 // check if segmentation fault on the node
316 //
317 char command0[1000];
318 char command[1000];
319 char *path = "segfault.log";
320 Long_t id,size,flags,modtime;
321
322 // first check if we have a log file
323 sprintf(command0, Form("lsrun -m %s find /data.local2/proof/%s/ | grep %s | grep .log >filelist.list", machine,fUserName.Data(), filter));
324 if (fVerbose>1) {
325 printf("%s\n",command0);
326 }
327 gSystem->Exec(command0);
328 gSystem->GetPathInfo("filelist.list", &id, &size, &flags, &modtime);
329 if (size<1) return kFALSE;
330
331 //
332 // now check the content
333 //
334 sprintf(command, Form("lsrun -m %s cat `lsrun -m %s find /data.local2/proof/%s/ | grep %s|grep log ` | grep segmentation >segfault.log", machine, machine,fUserName.Data(), filter));
335
336 if (fVerbose) {
337 printf("%s\n",command);
338 }
339 if (fVerbose>1) gSystem->Exec(command);
340 gSystem->GetPathInfo(path, &id, &size, &flags, &modtime);
341 return (size>10);
342}
343
344
345//--------------------------------------------------------------------
346
347TTree * AliXRDPROOFtoolkit::DumpSys(Bool_t verbose){
348 //
349 // Dump memory usage on worker nodes
350 gSystem->Exec("echo > syswatch.log");
351 for(UInt_t i=0; i<listeMachine.size(); i++){
352 if (verbose) gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data()));
353 gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log >> syswatch.log",listeMachine[i]->Data()));
354 }
355
356 TTree * tree = new TTree("asci","asci");
357 tree->SetDirectory(0);
358 // tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F");
359 tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F");
360 return tree;
361}
362
363// //--------------------------------------------------------------------
364
365TTree * AliXRDPROOFtoolkit::DumpSys2(Bool_t verbose){
366 //
367 // Dump memory usage on worker nodes
368 //
369 gSystem->Exec("echo > syswatch.log");
370 for(UInt_t i=0; i<listeMachine.size(); i++){
371 if (verbose)
372 gSystem->Exec(Form("lsrun -m %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data()));
373 gSystem->Exec(Form("lsrun -m %s cat /data.local2/proof/miranov/last-workersession/syswatch.log >> syswatch.log",listeMachine[i]->Data()));
374 }
375
376 TTree * tree = new TTree("asci","asci");
377 tree->SetDirectory(0);
378 // tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F");
379 tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F");
380 return tree;
381}
382
383
384
385
386//
387
388
389TTree* AliXRDPROOFtoolkit::DumpFiles(Bool_t verbose){
390 //
391 //
392 //
393 gSystem->Exec("touch filewatch.log");
394 for(UInt_t i=0; i<listeMachine.size(); i++){
395 if (verbose) gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/worker*/filewatch.log >>filewatch.log",listeMachine[i]->Data()));
396 gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/worker*/filewatch.log >>filewatch.log",listeMachine[i]->Data()));
397 }
398 TTree * tree = new TTree("asci","asci");
399 tree->SetDirectory(0);
400 tree->ReadFile("filewatch.log","hname/C:fname/C:entry/I:status/I");
401 return tree;
402}
403
404
405
406
407
408//______________________________________________________________________________
409Int_t AliXRDPROOFtoolkit::Read(char * str, Int_t lenght, FILE *in)
410{
411 Int_t nread=0;
412 Int_t n;
413
414 while (lenght>0) {
415 n=fread(str+nread, sizeof(char), lenght, in);
416 if(n<=0) return nread;
417 nread+=n;
418 lenght-=n;
419 }
420 return nread;
421}
422
423
424
425
426//______________________________________________________________________________
427void AliXRDPROOFtoolkit::CheckFiles (const char*fileIn, UInt_t checkLevel, const char*treeToRetrieve, const char*varexp, const char*selection)
428{
429 //
430 // check the files
431 //
432 fstream fic;
433 fic.open(fileIn, ios_base::in);
434 fstream focGood;
435 fstream focBad;
436 focGood.open(Form("%s.Good",fileIn), ios_base::out|ios_base::trunc);
437 focBad.open(Form("%s.Bad",fileIn), ios_base::out|ios_base::trunc);
438 //
439 if(!fic.is_open()) {
440 cout<<"Can't open file "<<fileIn<<endl;
441 return;
442 }
443 //
444 //
445 //
446 Long64_t size;
447 char buffer[256];
448 char * line;
449 char * machine;
450 Int_t level=0;
451 Float_t compressionFactor;
452 Int_t nkey;
453 Int_t version;
454 TObjString * fileName=0x0;
455 TObjString * machineName=0x0;
456 // TChain chain ("check_chain");
457
458 TFile fout(Form("%s.root",fileIn),"recreate");
459 TTree * tree=new TTree("stats", "stats of AliXRDPROOFtoolkit::CheckFiles function");
460
461 tree->Branch ("machine", "TObjString", &machineName, 256000,0);
462 tree->Branch ("file", "TObjString", &fileName, 256000,0);
463 tree->Branch ("level", &level, "level/I");
464 tree->Branch ("size", &size, "size/L");
465 tree->Branch ("nkey", &nkey, "nkey/I");
466 tree->Branch ("compress", &compressionFactor, "compress/F");
467 tree->Branch ("version", &version, "version/I");
468
469 // start loop over the files
470 fic.getline(buffer, sizeof(buffer));
471 while (fic.good()){
472
473 // get the machine name if present in the file
474 machine=strchr(buffer, '\t');
475 if(machine!=0x0){
476 machine[0]='\0';
477 line=machine+1;
478 machine=buffer;
479 machineName=new TObjString(machine);
480 }else {
481 machineName=new TObjString("x");
482 line=buffer;
483 }
484 cout<<"Inspecting file :"<<line<<endl;
485
486
487 TTree * getTree=0x0;
488 fileName=new TObjString(line);
489
490 TFile * f=TFile::Open(line);
491 // chain.AddFile(line);
492 level=0;
493 size=-1;
494 nkey=-1;
495 compressionFactor=-1;
496 version=-1;
497
498 if (fileName->String().Contains("AliESDs.root")){
499 //
500 // check friend file
501 //
502 char fnameFriend[1000];
503 sprintf(fnameFriend,"%s/AliESDfriends.root",gSystem->DirName(fileName->String().Data()));
504 cout<<"Inspecting file :"<<fnameFriend<<endl;
505 TFile * ffriend=TFile::Open(fnameFriend);
506 if (!ffriend) level=-4;
507 else{
508 if (ffriend->IsZombie()) level=-3;
509 if (ffriend->TestBit(TFile::kRecovered) || ffriend->TestBit(TFile::kWriteError)){
510 level=-2;
511 }
512 ffriend->Close();
513 delete ffriend;
514 }
515 }
516
517
518 if(level>=-1 && f!=0x0){
519 level=1;
520 size=f->GetSize();
521 nkey=f->GetNkeys();
522 compressionFactor=f->GetCompressionFactor();
523 version=f->GetVersion();
524
525 if(checkLevel>0 && !f->IsZombie()){
526 level=2;
527 if(checkLevel>1 && treeToRetrieve!="" && (getTree=(TTree*)f->Get(treeToRetrieve))!=0x0){
528 level=3;
529 Int_t tentries = getTree->GetEntries();
530 if (tentries>=0) level=4;
531 cout<<"Number of entries :"<<getTree->GetEntries()<<endl;
532
533 if(checkLevel>3 && varexp!="" &&tentries>0) {
534 getTree->SetBranchStatus("*",1);
535 try{
536 TH1F his("his","his",100,-1,1);
537 Int_t selected = getTree->Draw(Form("%s>>his",varexp), selection, "goff", 1, tentries-1);
538 cout<<"Selected\t"<<selected<<endl;
539 if(selected>-1){
540 level=5;
541
542 //try to remove the created histogrames ...
543// TH1F *htemp = (TH1F*)gPad->GetPrimitive("htemp"); // 1D
544// TGraph *graph = (TGraph*)gPad->GetPrimitive("Graph"); // 2D
545// if(htemp!=0x0) {cout<<"removing TH1D"<<endl; delete htemp;}
546// else if(graph!=0x0) {cout<<"removing TGraph"<<endl; delete graph;}
547// else cout<<"nothing to remove : memory leak ..."<<endl;
548
549 }
550 }catch(std::bad_alloc){
551 cout<<"Warning : Draw option send std::badalloc"<<endl;
552 }
553 }
554 delete getTree;
555 }
556 }
557 f->Close();
558 delete f;
559 }
560 if (level>checkLevel){
561 focGood<<line<<endl;
562 }
563 else{
564 focBad<<line<<"\t"<<level<<endl;
565 }
566
567
568 tree->Fill();
569 fic.getline(buffer, sizeof(buffer));
570 }
571
572 //now use chain to check
573 //chain.Lookup(kTRUE);
574
575
576 // Save the tree
577 tree->Write();
578 fout.Close();
579 focGood.close();
580 focBad.close();
581
582}
583
584
585
586
587