[u/mrichter/AliRoot.git] / TPC / macros / AliXRDPROOFtoolkit.cxx

/*
  TTOOLKIT to acces files on XRD 
  WORKING ONLY AT GSI
//

  Example usage:
  Load toolkit
  gSystem->AddIncludePath("-I$ALICE_ROOT/TPC/macros")
  gROOT->LoadMacro("$ALICE_ROOT/TPC/macros/AliXRDPROOFtoolkit.cxx++")

  1. 
  Retrieve the list of files : POSSIBLE ONLY FOR USER WITH LOGIN ACCESS TO THE XRD MACHINES 

  AliXRDPROOFtoolkit tool;
  tool.ListOfFiles("pp.txt","/data.local2/sma/sim/v4-05-Rev-03/pp", "AliESDs.root", kTRUE);
  
  // check the list
  AliXRDPROOFtoolkit::FilterList("pp.txt","AliESDs.root esdTree",0)  
  2. 
  
  AliXRDPROOFtoolkit tool;
  TChain * chain = tool.MakeChain("pp.txt","esdTree",10)
  chain->Draw("fTPCncls");

  
  3. Process logs - ONLY priority users - with ssh acces
  AliXRDPROOFtoolkit tool;
  tool.FilterSegFault();
  TTree * treeSys  = tool.DumpSys(kTRUE)

*/


#include <TTree.h>
#include <TString.h>
#include <TObjArray.h>
#include <TObjString.h>
#include <TTree.h>
#include <TFile.h>
#include <TChain.h>
#include <TDSet.h>
#include <TH1.h>
#include <TGraph.h>
#include <TMath.h>
#include <TPad.h>
#include <exception>
#include <fstream>
#include <TRandom.h>


#include <AliXRDPROOFtoolkit.h>


ClassImp(AliXRDPROOFtoolkit);


//______________________________________________________________________________
AliXRDPROOFtoolkit::AliXRDPROOFtoolkit () : TObject () 
{
  //
  // Default  -GSI specific setup
  //
  for(Int_t i=256;i<268;i++)
    listeMachine.push_back(new TString(Form("lxb%d.gsi.de", i)));
  fUserGroup = gSystem->GetUserInfo();
  fUserName = fUserGroup->fUser;       
  fVerbose=1;
}

void AliXRDPROOFtoolkit::Print(Option_t* /*option*/) const{
  //
  // Print current setup of toolkit
  //
  printf("User name\t%s\n", fUserName.Data());
  printf("List of slaves\n");

  for(UInt_t i=0;i<listeMachine.size();i++)
    printf("%s\n",listeMachine[i]->Data());
}


//______________________________________________________________________________
void AliXRDPROOFtoolkit::AddMachine (const char*name)
{
  //
  // add computer to the list
  //
  listeMachine.push_back( new TString(name));
}


//______________________________________________________________________________
Bool_t AliXRDPROOFtoolkit::ListOfFiles(const char*fileName, const char*path, const char*filter, Bool_t displayMachine)
{
  //
  // Get the list of files on "registerd slaves"
  //
  // fileName - Resultinfg file with list
  // path     - starting path on slave e.g /data.local2/sma/
  // filter   - filter expression e.g. AliESDs.root
  // display machine - not used yet

  char filterXRD[100];
  char command[1000];
  //
  // FILTER expression - to get REDIRECTOR NAME
  //
  //what we want for sed
  //  sprintf(filterXRD,"sed s/\\/data.local2/root:\\/\\/gsiaf.gsi.de:1094\\//"); 
  //what we should write
  sprintf(filterXRD,"sed s/\\\\/data.local2/root:\\\\/\\\\/gsiaf.gsi.de:1094\\\\//");
  //
  // 
  //
  gSystem->Exec(Form("echo  >%s",fileName));
  for(UInt_t i=0; i<listeMachine.size(); i++){
    if (displayMachine){
      sprintf(filterXRD,"sed s/\\\\/data.local2/root:\\\\/\\\\/%s.gsi.de:1094\\\\//",listeMachine[i]->Data());
    }
    cout<<"Inspecting "<<listeMachine[i]->Data()<<" ..."<<endl;
    sprintf(command,"lsrun -m %s   find %s | grep %s | %s >> %s", listeMachine[i]->Data(), path, filter, filterXRD,fileName);
    printf(command);
    gSystem->Exec(command);    
  }
  return kTRUE;
}


TChain* AliXRDPROOFtoolkit::MakeChain(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles, Int_t startFile)
{
  //
  // Create the chain
  //
  TChain* chain = new TChain(treeName);

  // Open the input stream
  ifstream in;
  in.open(fileIn);

  // Read the input list of files and add them to the chain
  TString currentFile;
  Int_t counter=0;
  while(in.good()) {
    in >> currentFile;
    if (fName) {
      currentFile+="#";
      currentFile+=fName;
    }
    if (!currentFile.Contains("root")) continue; // protection
    counter++;
    if (counter<startFile) continue;
    if (counter>maxFiles+startFile) break;
    chain->Add(currentFile.Data());
  }

  in.close();

  return chain;
}

TChain* AliXRDPROOFtoolkit::MakeChainRandom(const char*fileIn, const char * treeName,const char *fName, Int_t maxFiles, Int_t startFile)
{
  //
  // Create a TDSet - files are in random order
  //
  // filein    - input list text file
  // treename  - containg tree 
  // maxFiles  - maximum number of files included

  TObjArray array(10000);
  
  TChain* chain = new TChain(treeName);

  // Open the input stream
  ifstream in;
  in.open(fileIn);

  // Read the input list of files and add them to the chain
  TString currentFile;
  Int_t counter=0;
  while(in.good()) {
    in >> currentFile;
    if (fName) {
      currentFile+="#";
      currentFile+=fName;
    }
    if (!currentFile.Contains("root")) continue; // protection
    counter++;
    //    chain->Add(currentFile.Data());
    array.AddLast(new TObjString(currentFile));
  }
  in.close();
  Int_t entries = array.GetEntries();
  printf("Number of entries\t%d",entries);
  if (maxFiles<0) maxFiles=entries;
  if (maxFiles>entries) maxFiles=entries;
  //
  //
  //
  for (Int_t i=0; i<maxFiles; i++){
    Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
    if (ifile<entries && (array.At(ifile)) &&  array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){ 
      printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
      chain->Add(array.At(ifile)->GetName());
      array.At(ifile)->SetBit(TObject::kCannotPick);
    }
  }
  return chain;
}


TDSet* AliXRDPROOFtoolkit::MakeSet(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
{
  //
  // Create the TDSet out of list
  // filein    - input list text file
  // treename  - containg tree 
  // maxFiles  - maximum number of files included

  TDSet* chain = new TDSet(treeName);

  // Open the input stream
  ifstream in;
  in.open(fileIn);

  // Read the input list of files and add them to the chain
  TString currentFile;
  Int_t counter=0;
  while(in.good()) {
    in >> currentFile;
    if (fName) {
      currentFile+="#";
      currentFile+=fName;
    }
    if (!currentFile.Contains("root")) continue; // protection
    counter++;
    if (maxFiles>0 && counter>maxFiles) break;
    chain->Add(currentFile.Data());
  }

  in.close();
  chain->Validate();
  return chain;
}


TDSet* AliXRDPROOFtoolkit::MakeSetRandom(const char*fileIn, const char * treeName, const char *fName, Int_t maxFiles)
{
  //
  // Create a TDSet - files are in random order
  //
  // filein    - input list text file
  // treename  - containg tree 
  // maxFiles  - maximum number of files included

  TObjArray array(10000);
  
  TDSet* chain = new TDSet(treeName);

  // Open the input stream
  ifstream in;
  in.open(fileIn);

  // Read the input list of files and add them to the chain
  TString currentFile;
  Int_t counter=0;
  while(in.good()) {
    in >> currentFile;
    if (fName) {
      currentFile+="#";
      currentFile+=fName;
    }
    if (!currentFile.Contains("root")) continue; // protection
    counter++;
    //    chain->Add(currentFile.Data());
    array.AddLast(new TObjString(currentFile));
  }
  in.close();
  Int_t entries = array.GetEntries();
  printf("Number of entries\t%d",entries);
  if (maxFiles<0) maxFiles=entries;
  if (maxFiles>entries) maxFiles=entries;
  for (Int_t i=0; i<maxFiles; i++){
    Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
    if (ifile<entries && (array.At(ifile)) &&  array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){ 
      printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
      chain->Add(array.At(ifile)->GetName());
      array.At(ifile)->SetBit(TObject::kCannotPick);
    }
  }


  chain->Validate();
  return chain;
}

//_______________________________________________________________________________


void   AliXRDPROOFtoolkit::FilterSegFault(const char *filter){
  //
  // Print the list of slaves finished with seg fault
  //  
  printf("List of the PROOF slaves  with seg fault (Last session) \n");
  for(UInt_t i=0; i<listeMachine.size(); i++){
    if (HasSegFault(listeMachine[i]->Data(), filter)){
      printf("%s\n",listeMachine[i]->Data());
    }
  }
}

//_______________________________________________________________________________

Bool_t    AliXRDPROOFtoolkit::HasSegFault(const char * machine, const char *filter){
  //
  // check if segmentation fault on the node
  //
  char command0[1000];
  char command[1000];
  char *path  =  "segfault.log";
  Long_t id,size,flags,modtime;
  
  // first check if we have a log file
  sprintf(command0, Form("lsrun -m %s find /data.local2/proof/%s/  | grep %s | grep .log >filelist.list", machine,fUserName.Data(), filter));
  if (fVerbose>1) {
    printf("%s\n",command0);
  }
  gSystem->Exec(command0);
  gSystem->GetPathInfo("filelist.list", &id, &size, &flags, &modtime);
  if (size<1) return kFALSE;
  
  //
  // now check the content
  //
  sprintf(command, Form("lsrun -m %s  cat `lsrun -m %s find /data.local2/proof/%s/  | grep %s|grep log ` | grep segmentation  >segfault.log", machine, machine,fUserName.Data(), filter));
  
  if (fVerbose) {
    printf("%s\n",command);
  }
  if (fVerbose>1) gSystem->Exec(command);
  gSystem->GetPathInfo(path, &id, &size, &flags, &modtime);
  return (size>10);
}


//--------------------------------------------------------------------

TTree *    AliXRDPROOFtoolkit::DumpSys(Bool_t verbose){
  //
  // Dump memory usage on worker nodes
  gSystem->Exec("echo > syswatch.log");
  for(UInt_t i=0; i<listeMachine.size(); i++){
    if (verbose) gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data()));
    gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log >> syswatch.log",listeMachine[i]->Data()));
  } 
  
  TTree * tree = new TTree("asci","asci"); 
  tree->SetDirectory(0);
  //  tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F");
  tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F");
  return tree;
}

// //--------------------------------------------------------------------

TTree * AliXRDPROOFtoolkit::DumpSys2(Bool_t verbose){
  //
  // Dump memory usage on worker nodes
  //
  gSystem->Exec("echo > syswatch.log");
  for(UInt_t i=0; i<listeMachine.size(); i++){
    if (verbose) 
      gSystem->Exec(Form("lsrun -m %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data()));
    gSystem->Exec(Form("lsrun -m  %s cat /data.local2/proof/miranov/last-workersession/syswatch.log >> syswatch.log",listeMachine[i]->Data()));
  } 
  
  TTree * tree = new TTree("asci","asci"); 
  tree->SetDirectory(0);
  //  tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F");
  tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F");
  return tree;
}


//


TTree*    AliXRDPROOFtoolkit::DumpFiles(Bool_t verbose){
  //
  //
  //
  gSystem->Exec("touch filewatch.log");
  for(UInt_t i=0; i<listeMachine.size(); i++){ 
    if (verbose) gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/worker*/filewatch.log >>filewatch.log",listeMachine[i]->Data()));
    gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/worker*/filewatch.log >>filewatch.log",listeMachine[i]->Data()));
  } 
 TTree * tree = new TTree("asci","asci"); 
 tree->SetDirectory(0);
 tree->ReadFile("filewatch.log","hname/C:fname/C:entry/I:status/I");
 return tree;  
}


//______________________________________________________________________________
void AliXRDPROOFtoolkit::CheckFiles (const char*fileIn, UInt_t checkLevel, const char*treeToRetrieve, const char*varexp, const char*selection)
{
  //
  // check the files
  //  
  fstream fic;
  fic.open(fileIn, ios_base::in);
  fstream focGood;
  fstream focBad;
  focGood.open(Form("%s.Good",fileIn), ios_base::out|ios_base::trunc);
  focBad.open(Form("%s.Bad",fileIn), ios_base::out|ios_base::trunc);
  //
  if(!fic.is_open()) {
    cout<<"Can't open file "<<fileIn<<endl;
    return;
  }
  //
  //
  //
  Long64_t size;
  char buffer[256];
  char * line;
  char * machine;
  Int_t level=0;
  Float_t compressionFactor;
  Int_t nkey;
  Int_t version;
  TObjString * fileName=0x0;
  TObjString * machineName=0x0;
  //  TChain chain ("check_chain");
  
  TFile fout(Form("%s.root",fileIn),"recreate");
  TTree * tree=new TTree("stats", "stats of AliXRDPROOFtoolkit::CheckFiles function");
  
  tree->Branch ("machine", "TObjString", &machineName, 256000,0);
  tree->Branch ("file", "TObjString", &fileName, 256000,0);
  tree->Branch ("level", &level, "level/I");
  tree->Branch ("size", &size, "size/L");
  tree->Branch ("nkey", &nkey, "nkey/I");
  tree->Branch ("compress", &compressionFactor, "compress/F");
  tree->Branch ("version", &version, "version/I");

  // start loop over the files
  fic.getline(buffer, sizeof(buffer));
  while (fic.good()){

    // get the machine name if present in the file
    machine=strchr(buffer, '\t');
    if(machine!=0x0){
      machine[0]='\0';
      line=machine+1;
      machine=buffer;
      machineName=new TObjString(machine);
    }else {
      machineName=new TObjString("x");
      line=buffer;
    }
    cout<<"Inspecting file :"<<line<<endl;  

 
    TTree * getTree=0x0;
    fileName=new TObjString(line);
    
    TFile * f=TFile::Open(line);
    //    chain.AddFile(line);
    level=0;
    size=-1;
    nkey=-1;
    compressionFactor=-1;
    version=-1;
    
    if (fileName->String().Contains("AliESDs.root")){
      //
      // check  friend file 
      //
      char  fnameFriend[1000];
      sprintf(fnameFriend,"%s/AliESDfriends.root",gSystem->DirName(fileName->String().Data()));
      cout<<"Inspecting file :"<<fnameFriend<<endl;  
      TFile * ffriend=TFile::Open(fnameFriend);
      if (!ffriend) level=-4;
      else{
	if (ffriend->IsZombie()) level=-3;
	if (ffriend->TestBit(TFile::kRecovered) || ffriend->TestBit(TFile::kWriteError)){
	  level=-2;
	}
	ffriend->Close();
	delete ffriend;
      }
    }
    

    if(level>=-1 && f!=0x0){
      level=1;
      size=f->GetSize();
      nkey=f->GetNkeys();
      compressionFactor=f->GetCompressionFactor();
      version=f->GetVersion();

      if(checkLevel>0 && !f->IsZombie()){
	level=2;
	if(checkLevel>1 && treeToRetrieve!="" && (getTree=(TTree*)f->Get(treeToRetrieve))!=0x0){
	  level=3;
	  Int_t tentries = getTree->GetEntries();
	  if (tentries>=0) level=4;
	  cout<<"Number of entries :"<<getTree->GetEntries()<<endl;  
	  
	  if(checkLevel>3 && varexp!="" &&tentries>0) {
	    getTree->SetBranchStatus("*",1);	    
	    try{
	      TH1F his("his","his",100,-1,1);
	      Int_t selected = getTree->Draw(Form("%s>>his",varexp), selection, "goff", 1, tentries-1);
	      cout<<"Selected\t"<<selected<<endl;
	      if(selected>-1){
		level=5;
		
		//try to remove the created histogrames ...
// 		TH1F *htemp = (TH1F*)gPad->GetPrimitive("htemp"); // 1D
// 		TGraph *graph = (TGraph*)gPad->GetPrimitive("Graph"); // 2D
// 		if(htemp!=0x0) {cout<<"removing TH1D"<<endl; delete htemp;}
// 		else if(graph!=0x0) {cout<<"removing TGraph"<<endl; delete graph;}
// 		else cout<<"nothing to remove : memory leak ..."<<endl;

	      }
	    }catch(std::bad_alloc){
	      cout<<"Warning : Draw option send std::badalloc"<<endl;
	    }
	  }
	  delete getTree;
	}
      }
      f->Close();
      delete f;
    }
    if (level>checkLevel){
      focGood<<line<<endl;
    }
    else{
      focBad<<line<<"\t"<<level<<endl;
    }

      
    tree->Fill();
    fic.getline(buffer, sizeof(buffer)); 
  }

  //now use chain to check
  //chain.Lookup(kTRUE);


  // Save the tree
  tree->Write();
  fout.Close();
  focGood.close();
  focBad.close();
  
}

Int_t  AliXRDPROOFtoolkit::CheckTreeInFile(const char*fileName,const char*treeName, Int_t debugLevel, const char *branchName){
  //
  // Check the tree in file 
  // fileName   - the name of the file with tree
  // treeName   - the name of file
  // debugLevel - 0 check the existance of the file -  1 make loop over entries
  // branchName - if debugLevel>0 the branch is chcecked
  //              if brnachName =0 the content of full tree is chcecked
  // return value = 0 - Check things  OK
  //               -1 - file not exist or not accesible
  //               -2 - file is zombie
  //		   -3 - tree not present
  //               -4 - branch not present
  TFile * file = TFile::Open(fileName);
  if (!file) return -1;
  if (file->IsZombie()) {delete file; return -2;};
  if (treeName="*") return 0;
  TTree * tree = (TTree*)file->Get(treeName);
  if (!tree) return -3;
  TBranch * branch = 0;
  if (branchName) {
    branch = tree->GetBranch(branchName);
    if (!branch) return -4;
  }
  //
  tree->SetBranchStatus("*",1);
  try {
    if (debugLevel>0){
      Int_t entries = tree->GetEntries();
      for (Int_t i=0;i<entries; i++){
	if (branch) branch->GetEntry(i);
	else tree->GetEntry();      
      }
    }
  }catch ( ... ) {
    printf("PROBLEM\n");  
    // never catched  - as there is no exception in the ROOT IO
    return 1 ;
  }

  return 0;
}


Bool_t  AliXRDPROOFtoolkit::FilterList(const char*inputList, const char*fileList, Int_t checkLevel){
  //
  // Filter the list  
  // inputList - list of original file names
  // fileList  - list of file to be checked
  //           - 0 - fileName
  //           - 1 - treeName (if * not checked)
  //           - 2 - fileName 
  //                 ....
  // checkLevel - 0 - check only existance of the files and tree's + 
  //                  simple file corruption
  //            > 1 - check the content of the tree - 
  //                  (can crash as there do not exest exception handling in ROOT)
  // Output -  two streams are created - file with good entries
  // "%s.Good a,d file with bad entries %s.Bad
  //EXAMPLE:
  // AliXRDPROOFtoolkit::FilterList("ppgrid2.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",1) 

  fstream finput;
  finput.open(inputList, ios_base::in);
  fstream focGood;
  fstream focBad;
  focGood.open(Form("%s.Good",inputList), ios_base::out|ios_base::trunc);
  focBad.open(Form("%s.Bad",inputList), ios_base::out|ios_base::trunc);
  //
  if(!finput.is_open()) {
    cout<<"Can't open file "<<inputList<<endl;
    return kFALSE;
  }
  //
  // Read the input list of files and add them to the chain
  //
  TObjArray *array = (TString(fileList)).Tokenize(" ");
  TString currentFile;
  Int_t counter=0;
  while(finput.good()) {
    finput >> currentFile;
    if (!currentFile.Contains("root")) continue; // protection
    //    Bool_t isZip = currentFile.Contains("#");
    const char * dirname = gSystem->DirName(currentFile.Data());
    Int_t status = 0;
    for (Int_t i=0; i<array->GetEntries(); i+=2){
      char fname[1000];
      //if (isZip) sprintf(fname,
      sprintf(fname, "%s/%s",dirname,array->At(i)->GetName());
      Int_t cstatus = CheckTreeInFile(fname, array->At(i+1)->GetName(), checkLevel,0);
      if (cstatus!=0) {
	status = cstatus; 
	break;
      }
    }
    if (status==0){
      focGood<<currentFile<<endl;
    }else{
      focBad<<currentFile<<endl;
    }
    counter++;    
  }
  finput.close();
}


Bool_t  AliXRDPROOFtoolkit::XRDCopyDir(const char * idir, const char * files, const char *odir, Bool_t zip){
  //
  // idir  - input directory
  // odir  - output directory
  // files - the list of files to be coppied
  // zip   - not supported yet
  //
  // Example :									
  //
  // idir ="root://gsiaf.gsi.de:1094//sma/sim/v4-05-Rev-03/pp/0000";
  // odir ="root://lxgrid2.gsi.de:1094//miranov/test/pp/0000"; 
  // char *files="AliESDs.root AliESDfriend.root Kinematics.root";
  TString str(files);
  TObjArray * array = str.Tokenize(" "); 
  Int_t nfiles = array->GetEntries();
  char infile[1000];
  char outfile[1000];
  Bool_t succes=kTRUE;
  for (Int_t ifile =0; ifile<nfiles; ifile++){
    sprintf(infile,"%s/%s", idir, array->At(ifile)->GetName());
    sprintf(outfile,"%s/%s", odir, array->At(ifile)->GetName());
    printf("%s - %s\n",infile, outfile);
    Bool_t result = TFile::Cp(infile,outfile); 
    succes &= result;
  }
  return succes;
}
Commit	Line	Data
db6fdd20	1	/*
	2	TTOOLKIT to acces files on XRD
	3	WORKING ONLY AT GSI
	4	//
	5
	6	Example usage:
	7	Load toolkit
	8	gSystem->AddIncludePath("-I$ALICE_ROOT/TPC/macros")
	9	gROOT->LoadMacro("$ALICE_ROOT/TPC/macros/AliXRDPROOFtoolkit.cxx++")
	10
	11	1.
	12	Retrieve the list of files : POSSIBLE ONLY FOR USER WITH LOGIN ACCESS TO THE XRD MACHINES
	13
	14	AliXRDPROOFtoolkit tool;
	15	tool.ListOfFiles("pp.txt","/data.local2/sma/sim/v4-05-Rev-03/pp", "AliESDs.root", kTRUE);
18519053	16
	17	// check the list
	18	AliXRDPROOFtoolkit::FilterList("pp.txt","AliESDs.root esdTree",0)
db6fdd20	19	2.
	20
	21	AliXRDPROOFtoolkit tool;
	22	TChain * chain = tool.MakeChain("pp.txt","esdTree",10)
	23	chain->Draw("fTPCncls");
	24
	25
	26	3. Process logs - ONLY priority users - with ssh acces
	27	AliXRDPROOFtoolkit tool;
	28	tool.FilterSegFault();
	29	TTree * treeSys = tool.DumpSys(kTRUE)
	30
	31	*/
	32
	33
	34	#include <TTree.h>
	35	#include <TString.h>
	36	#include <TObjArray.h>
	37	#include <TObjString.h>
	38	#include <TTree.h>
	39	#include <TFile.h>
	40	#include <TChain.h>
	41	#include <TDSet.h>
	42	#include <TH1.h>
	43	#include <TGraph.h>
	44	#include <TMath.h>
	45	#include <TPad.h>
	46	#include <exception>
	47	#include <fstream>
	48	#include <TRandom.h>
	49
	50
	51	#include <AliXRDPROOFtoolkit.h>
	52
	53
	54	ClassImp(AliXRDPROOFtoolkit);
	55
	56
	57
	58	//______________________________________________________________________________
	59	AliXRDPROOFtoolkit::AliXRDPROOFtoolkit () : TObject ()
	60	{
	61	//
	62	// Default -GSI specific setup
	63	//
	64	for(Int_t i=256;i<268;i++)
	65	listeMachine.push_back(new TString(Form("lxb%d.gsi.de", i)));
	66	fUserGroup = gSystem->GetUserInfo();
	67	fUserName = fUserGroup->fUser;
	68	fVerbose=1;
	69	}
	70
	71	void AliXRDPROOFtoolkit::Print(Option_t* /option/) const{
	72	//
	73	// Print current setup of toolkit
	74	//
	75	printf("User name\t%s\n", fUserName.Data());
	76	printf("List of slaves\n");
	77
	78	for(UInt_t i=0;i<listeMachine.size();i++)
	79	printf("%s\n",listeMachine[i]->Data());
	80	}
	81
	82
83	//______________________________________________________________________________
84	void AliXRDPROOFtoolkit::AddMachine (const char*name)
85	{
86	//
87	// add computer to the list
88	//
89	listeMachine.push_back( new TString(name));
90	}
91
92
93	//______________________________________________________________________________
94	Bool_t AliXRDPROOFtoolkit::ListOfFiles(const charfileName, const charpath, const char*filter, Bool_t displayMachine)
95	{
96	//
97	// Get the list of files on "registerd slaves"
98	//
18519053	99	// fileName - Resultinfg file with list
db6fdd20	100	// path - starting path on slave e.g /data.local2/sma/
	101	// filter - filter expression e.g. AliESDs.root
	102	// display machine - not used yet
	103
	104	char filterXRD[100];
	105	char command[1000];
	106	//
	107	// FILTER expression - to get REDIRECTOR NAME
	108	//
	109	//what we want for sed
	110	// sprintf(filterXRD,"sed s/\\/data.local2/root:\\/\\/gsiaf.gsi.de:1094\\//");
	111	//what we should write
	112	sprintf(filterXRD,"sed s/\\\\/data.local2/root:\\\\/\\\\/gsiaf.gsi.de:1094\\\\//");
	113	//
	114	//
	115	//
	116	gSystem->Exec(Form("echo >%s",fileName));
	117	for(UInt_t i=0; i<listeMachine.size(); i++){
18519053	118	if (displayMachine){
	119	sprintf(filterXRD,"sed s/\\\\/data.local2/root:\\\\/\\\\/%s.gsi.de:1094\\\\//",listeMachine[i]->Data());
	120	}
db6fdd20	121	cout<<"Inspecting "<<listeMachine[i]->Data()<<" ..."<<endl;
	122	sprintf(command,"lsrun -m %s find %s \| grep %s \| %s >> %s", listeMachine[i]->Data(), path, filter, filterXRD,fileName);
	123	printf(command);
	124	gSystem->Exec(command);
	125	}
	126	return kTRUE;
	127	}
	128
	129
	130
	131	TChain* AliXRDPROOFtoolkit::MakeChain(const charfileIn, const char treeName, const char *fName, Int_t maxFiles, Int_t startFile)
	132	{
	133	//
	134	// Create the chain
	135	//
	136	TChain* chain = new TChain(treeName);
	137
	138	// Open the input stream
	139	ifstream in;
	140	in.open(fileIn);
	141
	142	// Read the input list of files and add them to the chain
	143	TString currentFile;
	144	Int_t counter=0;
	145	while(in.good()) {
	146	in >> currentFile;
	147	if (fName) {
	148	currentFile+="#";
	149	currentFile+=fName;
	150	}
	151	if (!currentFile.Contains("root")) continue; // protection
	152	counter++;
	153	if (counter<startFile) continue;
	154	if (counter>maxFiles+startFile) break;
	155	chain->Add(currentFile.Data());
	156	}
	157
	158	in.close();
	159
	160	return chain;
	161	}
	162
	163	TChain* AliXRDPROOFtoolkit::MakeChainRandom(const charfileIn, const char treeName,const char *fName, Int_t maxFiles, Int_t startFile)
	164	{
	165	//
	166	// Create a TDSet - files are in random order
	167	//
	168	// filein - input list text file
	169	// treename - containg tree
	170	// maxFiles - maximum number of files included
	171
	172	TObjArray array(10000);
	173
	174	TChain* chain = new TChain(treeName);
	175
	176	// Open the input stream
	177	ifstream in;
	178	in.open(fileIn);
	179
	180	// Read the input list of files and add them to the chain
	181	TString currentFile;
	182	Int_t counter=0;
	183	while(in.good()) {
	184	in >> currentFile;
185	if (fName) {
186	currentFile+="#";
187	currentFile+=fName;
188	}
189	if (!currentFile.Contains("root")) continue; // protection
190	counter++;
191	// chain->Add(currentFile.Data());
192	array.AddLast(new TObjString(currentFile));
193	}
194	in.close();
195	Int_t entries = array.GetEntries();
196	printf("Number of entries\t%d",entries);
197	if (maxFiles<0) maxFiles=entries;
198	if (maxFiles>entries) maxFiles=entries;
199	//
200	//
201	//
202	for (Int_t i=0; i<maxFiles; i++){
203	Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
204	if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
205	printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
206	chain->Add(array.At(ifile)->GetName());
207	array.At(ifile)->SetBit(TObject::kCannotPick);
208	}
209	}
210	return chain;
211	}
212
213
214
215	TDSet* AliXRDPROOFtoolkit::MakeSet(const charfileIn, const char treeName, const char *fName, Int_t maxFiles)
216	{
217	//
218	// Create the TDSet out of list
219	// filein - input list text file
220	// treename - containg tree
221	// maxFiles - maximum number of files included
222
223	TDSet* chain = new TDSet(treeName);
224
225	// Open the input stream
226	ifstream in;
227	in.open(fileIn);
228
229	// Read the input list of files and add them to the chain
230	TString currentFile;
231	Int_t counter=0;
232	while(in.good()) {
233	in >> currentFile;
234	if (fName) {
235	currentFile+="#";
236	currentFile+=fName;
237	}
238	if (!currentFile.Contains("root")) continue; // protection
239	counter++;
240	if (maxFiles>0 && counter>maxFiles) break;
241	chain->Add(currentFile.Data());
242	}
243
244	in.close();
245	chain->Validate();
246	return chain;
247	}
248
249
250	TDSet* AliXRDPROOFtoolkit::MakeSetRandom(const charfileIn, const char treeName, const char *fName, Int_t maxFiles)
251	{
252	//
253	// Create a TDSet - files are in random order
254	//
255	// filein - input list text file
256	// treename - containg tree
257	// maxFiles - maximum number of files included
258
259	TObjArray array(10000);
260
261	TDSet* chain = new TDSet(treeName);
262
263	// Open the input stream
264	ifstream in;
265	in.open(fileIn);
266
267	// Read the input list of files and add them to the chain
268	TString currentFile;
269	Int_t counter=0;
270	while(in.good()) {
271	in >> currentFile;
272	if (fName) {
273	currentFile+="#";
274	currentFile+=fName;
275	}
276	if (!currentFile.Contains("root")) continue; // protection
277	counter++;
278	// chain->Add(currentFile.Data());
279	array.AddLast(new TObjString(currentFile));
280	}
281	in.close();
282	Int_t entries = array.GetEntries();
283	printf("Number of entries\t%d",entries);
284	if (maxFiles<0) maxFiles=entries;
285	if (maxFiles>entries) maxFiles=entries;
286	for (Int_t i=0; i<maxFiles; i++){
287	Int_t ifile = TMath::Nint(gRandom->Rndm()*Float_t(entries));
288	if (ifile<entries && (array.At(ifile)) && array.At(ifile)->TestBit(TObject::kCannotPick)==kFALSE){
289	printf("%d\t%d\t%s\n",i, ifile, array.At(ifile)->GetName());
290	chain->Add(array.At(ifile)->GetName());
291	array.At(ifile)->SetBit(TObject::kCannotPick);
292	}
293	}
294
295
296	chain->Validate();
297	return chain;
298	}
299
300	//_______________________________________________________________________________
301
302
303	void AliXRDPROOFtoolkit::FilterSegFault(const char *filter){
304	//
305	// Print the list of slaves finished with seg fault
306	//
307	printf("List of the PROOF slaves with seg fault (Last session) \n");
308	for(UInt_t i=0; i<listeMachine.size(); i++){
309	if (HasSegFault(listeMachine[i]->Data(), filter)){
310	printf("%s\n",listeMachine[i]->Data());
311	}
312	}
313	}
314
315	//_______________________________________________________________________________
316
317	Bool_t AliXRDPROOFtoolkit::HasSegFault(const char * machine, const char *filter){
318	//
319	// check if segmentation fault on the node
320	//
321	char command0[1000];
322	char command[1000];
323	char *path = "segfault.log";
324	Long_t id,size,flags,modtime;
325
326	// first check if we have a log file
327	sprintf(command0, Form("lsrun -m %s find /data.local2/proof/%s/ \| grep %s \| grep .log >filelist.list", machine,fUserName.Data(), filter));
328	if (fVerbose>1) {
329	printf("%s\n",command0);
330	}
331	gSystem->Exec(command0);
332	gSystem->GetPathInfo("filelist.list", &id, &size, &flags, &modtime);
333	if (size<1) return kFALSE;
334
335	//
336	// now check the content
337	//
338	sprintf(command, Form("lsrun -m %s cat `lsrun -m %s find /data.local2/proof/%s/ \| grep %s\|grep log ` \| grep segmentation >segfault.log", machine, machine,fUserName.Data(), filter));
339
340	if (fVerbose) {
341	printf("%s\n",command);
342	}
343	if (fVerbose>1) gSystem->Exec(command);
344	gSystem->GetPathInfo(path, &id, &size, &flags, &modtime);
345	return (size>10);
346	}
347
348
349	//--------------------------------------------------------------------
350
351	TTree * AliXRDPROOFtoolkit::DumpSys(Bool_t verbose){
352	//
353	// Dump memory usage on worker nodes
354	gSystem->Exec("echo > syswatch.log");
355	for(UInt_t i=0; i<listeMachine.size(); i++){
356	if (verbose) gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data()));
357	gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker*/syswatch.log >> syswatch.log",listeMachine[i]->Data()));
358	}
359
360	TTree * tree = new TTree("asci","asci");
361	tree->SetDirectory(0);
362	// tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F");
363	tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F");
364	return tree;
365	}
366
367	// //--------------------------------------------------------------------
368
369	TTree * AliXRDPROOFtoolkit::DumpSys2(Bool_t verbose){
370	//
371	// Dump memory usage on worker nodes
372	//
373	gSystem->Exec("echo > syswatch.log");
374	for(UInt_t i=0; i<listeMachine.size(); i++){
375	if (verbose)
376	gSystem->Exec(Form("lsrun -m %s cat /data.local2/proof/miranov/last-worker*/syswatch.log ",listeMachine[i]->Data()));
377	gSystem->Exec(Form("lsrun -m %s cat /data.local2/proof/miranov/last-workersession/syswatch.log >> syswatch.log",listeMachine[i]->Data()));
378	}
379
380	TTree * tree = new TTree("asci","asci");
381	tree->SetDirectory(0);
382	// tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F");
383	tree->ReadFile("syswatch.log","hname/C:fname/C:entry/I:time/I:mem0/F:mem1/F:mem2/F:mem3/F:cpu0/F:cpu1/F:cpu2/F:cpu3/F");
384	return tree;
385	}
386
387
388
389
390	//
391
392
393	TTree* AliXRDPROOFtoolkit::DumpFiles(Bool_t verbose){
394	//
395	//
396	//
397	gSystem->Exec("touch filewatch.log");
398	for(UInt_t i=0; i<listeMachine.size(); i++){
399	if (verbose) gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker/worker/filewatch.log >>filewatch.log",listeMachine[i]->Data()));
400	gSystem->Exec(Form("ssh %s cat /data.local2/proof/miranov/last-worker/worker/filewatch.log >>filewatch.log",listeMachine[i]->Data()));
401	}
402	TTree * tree = new TTree("asci","asci");
403	tree->SetDirectory(0);
404	tree->ReadFile("filewatch.log","hname/C:fname/C:entry/I:status/I");
405	return tree;
406	}
407
408
409
410
411
db6fdd20	412
	413	//______________________________________________________________________________
	414	void AliXRDPROOFtoolkit::CheckFiles (const charfileIn, UInt_t checkLevel, const chartreeToRetrieve, const charvarexp, const charselection)
	415	{
	416	//
	417	// check the files
	418	//
	419	fstream fic;
	420	fic.open(fileIn, ios_base::in);
	421	fstream focGood;
	422	fstream focBad;
	423	focGood.open(Form("%s.Good",fileIn), ios_base::out\|ios_base::trunc);
	424	focBad.open(Form("%s.Bad",fileIn), ios_base::out\|ios_base::trunc);
	425	//
	426	if(!fic.is_open()) {
	427	cout<<"Can't open file "<<fileIn<<endl;
	428	return;
	429	}
	430	//
	431	//
	432	//
	433	Long64_t size;
	434	char buffer[256];
	435	char * line;
	436	char * machine;
	437	Int_t level=0;
	438	Float_t compressionFactor;
	439	Int_t nkey;
	440	Int_t version;
	441	TObjString * fileName=0x0;
	442	TObjString * machineName=0x0;
	443	// TChain chain ("check_chain");
	444
	445	TFile fout(Form("%s.root",fileIn),"recreate");
	446	TTree * tree=new TTree("stats", "stats of AliXRDPROOFtoolkit::CheckFiles function");
	447
	448	tree->Branch ("machine", "TObjString", &machineName, 256000,0);
	449	tree->Branch ("file", "TObjString", &fileName, 256000,0);
	450	tree->Branch ("level", &level, "level/I");
	451	tree->Branch ("size", &size, "size/L");
	452	tree->Branch ("nkey", &nkey, "nkey/I");
	453	tree->Branch ("compress", &compressionFactor, "compress/F");
	454	tree->Branch ("version", &version, "version/I");
	455
	456	// start loop over the files
	457	fic.getline(buffer, sizeof(buffer));
	458	while (fic.good()){
	459
	460	// get the machine name if present in the file
	461	machine=strchr(buffer, '\t');
	462	if(machine!=0x0){
	463	machine[0]='\0';
	464	line=machine+1;
	465	machine=buffer;
	466	machineName=new TObjString(machine);
	467	}else {
	468	machineName=new TObjString("x");
	469	line=buffer;
	470	}
	471	cout<<"Inspecting file :"<<line<<endl;
	472
	473
	474	TTree * getTree=0x0;
	475	fileName=new TObjString(line);
476
477	TFile * f=TFile::Open(line);
478	// chain.AddFile(line);
479	level=0;
480	size=-1;
481	nkey=-1;
482	compressionFactor=-1;
483	version=-1;
484
485	if (fileName->String().Contains("AliESDs.root")){
486	//
487	// check friend file
488	//
489	char fnameFriend[1000];
490	sprintf(fnameFriend,"%s/AliESDfriends.root",gSystem->DirName(fileName->String().Data()));
491	cout<<"Inspecting file :"<<fnameFriend<<endl;
492	TFile * ffriend=TFile::Open(fnameFriend);
493	if (!ffriend) level=-4;
494	else{
495	if (ffriend->IsZombie()) level=-3;
496	if (ffriend->TestBit(TFile::kRecovered) \|\| ffriend->TestBit(TFile::kWriteError)){
497	level=-2;
498	}
499	ffriend->Close();
500	delete ffriend;
501	}
502	}
503
504
505	if(level>=-1 && f!=0x0){
506	level=1;
507	size=f->GetSize();
508	nkey=f->GetNkeys();
509	compressionFactor=f->GetCompressionFactor();
510	version=f->GetVersion();
511
512	if(checkLevel>0 && !f->IsZombie()){
513	level=2;
514	if(checkLevel>1 && treeToRetrieve!="" && (getTree=(TTree*)f->Get(treeToRetrieve))!=0x0){
515	level=3;
516	Int_t tentries = getTree->GetEntries();
517	if (tentries>=0) level=4;
518	cout<<"Number of entries :"<<getTree->GetEntries()<<endl;
519
520	if(checkLevel>3 && varexp!="" &&tentries>0) {
521	getTree->SetBranchStatus("*",1);
522	try{
523	TH1F his("his","his",100,-1,1);
524	Int_t selected = getTree->Draw(Form("%s>>his",varexp), selection, "goff", 1, tentries-1);
525	cout<<"Selected\t"<<selected<<endl;
526	if(selected>-1){
527	level=5;
528
529	//try to remove the created histogrames ...
530	// TH1F htemp = (TH1F)gPad->GetPrimitive("htemp"); // 1D
531	// TGraph graph = (TGraph)gPad->GetPrimitive("Graph"); // 2D
532	// if(htemp!=0x0) {cout<<"removing TH1D"<<endl; delete htemp;}
533	// else if(graph!=0x0) {cout<<"removing TGraph"<<endl; delete graph;}
534	// else cout<<"nothing to remove : memory leak ..."<<endl;
535
536	}
537	}catch(std::bad_alloc){
538	cout<<"Warning : Draw option send std::badalloc"<<endl;
539	}
540	}
541	delete getTree;
542	}
543	}
544	f->Close();
545	delete f;
546	}
547	if (level>checkLevel){
548	focGood<<line<<endl;
549	}
550	else{
551	focBad<<line<<"\t"<<level<<endl;
552	}
553
554
555	tree->Fill();
556	fic.getline(buffer, sizeof(buffer));
557	}
558
559	//now use chain to check
560	//chain.Lookup(kTRUE);
561
562
563	// Save the tree
564	tree->Write();
565	fout.Close();
566	focGood.close();
567	focBad.close();
0034e139	568
db6fdd20	569	}
db6fdd20	570
18519053	571	Int_t AliXRDPROOFtoolkit::CheckTreeInFile(const charfileName,const chartreeName, Int_t debugLevel, const char *branchName){
	572	//
	573	// Check the tree in file
	574	// fileName - the name of the file with tree
	575	// treeName - the name of file
	576	// debugLevel - 0 check the existance of the file - 1 make loop over entries
	577	// branchName - if debugLevel>0 the branch is chcecked
	578	// if brnachName =0 the content of full tree is chcecked
	579	// return value = 0 - Check things OK
	580	// -1 - file not exist or not accesible
	581	// -2 - file is zombie
	582	// -3 - tree not present
	583	// -4 - branch not present
	584	TFile * file = TFile::Open(fileName);
	585	if (!file) return -1;
	586	if (file->IsZombie()) {delete file; return -2;};
	587	if (treeName="*") return 0;
	588	TTree * tree = (TTree*)file->Get(treeName);
	589	if (!tree) return -3;
	590	TBranch * branch = 0;
	591	if (branchName) {
	592	branch = tree->GetBranch(branchName);
	593	if (!branch) return -4;
	594	}
	595	//
	596	tree->SetBranchStatus("*",1);
	597	try {
	598	if (debugLevel>0){
	599	Int_t entries = tree->GetEntries();
	600	for (Int_t i=0;i<entries; i++){
	601	if (branch) branch->GetEntry(i);
	602	else tree->GetEntry();
	603	}
	604	}
	605	}catch ( ... ) {
	606	printf("PROBLEM\n");
	607	// never catched - as there is no exception in the ROOT IO
	608	return 1 ;
	609	}
	610
	611	return 0;
	612	}
	613
	614
	615	Bool_t AliXRDPROOFtoolkit::FilterList(const charinputList, const charfileList, Int_t checkLevel){
	616	//
	617	// Filter the list
	618	// inputList - list of original file names
	619	// fileList - list of file to be checked
	620	// - 0 - fileName
	621	// - 1 - treeName (if * not checked)
	622	// - 2 - fileName
	623	// ....
	624	// checkLevel - 0 - check only existance of the files and tree's +
	625	// simple file corruption
	626	// > 1 - check the content of the tree -
	627	// (can crash as there do not exest exception handling in ROOT)
	628	// Output - two streams are created - file with good entries
	629	// "%s.Good a,d file with bad entries %s.Bad
	630	//EXAMPLE:
	631	// AliXRDPROOFtoolkit::FilterList("ppgrid2.txt","AliESDs.root esdTree AliESDfriends.root * Kinematics.root *",1)
	632
	633	fstream finput;
	634	finput.open(inputList, ios_base::in);
635	fstream focGood;
636	fstream focBad;
637	focGood.open(Form("%s.Good",inputList), ios_base::out\|ios_base::trunc);
638	focBad.open(Form("%s.Bad",inputList), ios_base::out\|ios_base::trunc);
639	//
640	if(!finput.is_open()) {
641	cout<<"Can't open file "<<inputList<<endl;
642	return kFALSE;
643	}
644	//
645	// Read the input list of files and add them to the chain
646	//
647	TObjArray *array = (TString(fileList)).Tokenize(" ");
648	TString currentFile;
649	Int_t counter=0;
650	while(finput.good()) {
651	finput >> currentFile;
652	if (!currentFile.Contains("root")) continue; // protection
653	// Bool_t isZip = currentFile.Contains("#");
654	const char * dirname = gSystem->DirName(currentFile.Data());
655	Int_t status = 0;
656	for (Int_t i=0; i<array->GetEntries(); i+=2){
657	char fname[1000];
658	//if (isZip) sprintf(fname,
659	sprintf(fname, "%s/%s",dirname,array->At(i)->GetName());
660	Int_t cstatus = CheckTreeInFile(fname, array->At(i+1)->GetName(), checkLevel,0);
661	if (cstatus!=0) {
662	status = cstatus;
663	break;
664	}
665	}
666	if (status==0){
667	focGood<<currentFile<<endl;
668	}else{
669	focBad<<currentFile<<endl;
670	}
671	counter++;
672	}
673	finput.close();
674	}
675
676
677
db6fdd20	678
db6fdd20	679
0034e139	680	Bool_t AliXRDPROOFtoolkit::XRDCopyDir(const char * idir, const char * files, const char *odir, Bool_t zip){
	681	//
	682	// idir - input directory
	683	// odir - output directory
	684	// files - the list of files to be coppied
	685	// zip - not supported yet
	686	//
	687	// Example :
	688	//
	689	// idir ="root://gsiaf.gsi.de:1094//sma/sim/v4-05-Rev-03/pp/0000";
	690	// odir ="root://lxgrid2.gsi.de:1094//miranov/test/pp/0000";
	691	// char *files="AliESDs.root AliESDfriend.root Kinematics.root";
	692	TString str(files);
	693	TObjArray * array = str.Tokenize(" ");
	694	Int_t nfiles = array->GetEntries();
	695	char infile[1000];
	696	char outfile[1000];
0034e139	697	Bool_t succes=kTRUE;
	698	for (Int_t ifile =0; ifile<nfiles; ifile++){
	699	sprintf(infile,"%s/%s", idir, array->At(ifile)->GetName());
	700	sprintf(outfile,"%s/%s", odir, array->At(ifile)->GetName());
	701	printf("%s - %s\n",infile, outfile);
	702	Bool_t result = TFile::Cp(infile,outfile);
	703	succes &= result;
	704	}
	705	return succes;
	706	}
	707
	708
db6fdd20	709
db6fdd20	710