[u/mrichter/AliRoot.git] / TPC / macros / testTPC / AlienToolkit.cxx

#include "TGrid.h"
#include "TGridCollection.h"
#include "TSystem.h"
#include "TString.h"
#include "TFile.h"
#include "TMap.h"
#include "TGridResult.h"
#include "TAlien.h"
#include "TAlienResult.h"
#include <fstream>


/*
  gSystem->Load("libXrdClient.so");
  gSystem->Load("libNetx.so");
  .L $ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx+
  //Reconstructed data example  
  char *mask = "v14*";
  char *path = "/alice/cern.ch/user/m/miranov/test2007/"
  // raw data
  char *mask = "0000191*RecPo*.root";
  char *path = "/alice/data/2008/LHC08a_TPC/"
 
  AlienToolkit toolkit;
  toolkit.MakeCollection(path,mask);

  //
  toolkit.MakeJobList("job.list","", "", "rec");

*/


class AlienToolkit: public TObject{
public:
  AlienToolkit();
  TGridCollection* MakeCollection(const char *path,  char *mask);
  void             Stage();
  void             LocalCopy(const char* destination);
  void             RemoteCopy(const char* destination="root://gsiaf.gsi.de:1094/", Int_t maxfiles=20);
  void             PrintPFN();
  void             MakeJobList(const  char * outname, const char *outputPrefix,  const char *action, const char *suffix);
  static Bool_t    IsDir(const char * name);
  static Bool_t    IsFile(const char * name);
  static Bool_t    ResubmitJobs();

public:
  TGridCollection *fCollection;
  TObjArray        fInfoArray;
  ClassDef(AlienToolkit,1)
};

ClassImp(AlienToolkit)

AlienToolkit::AlienToolkit():
  fCollection(0)
{
  if (!gGrid)
    TGrid::Connect("alien://",0,0,"t");
}


Bool_t AlienToolkit::IsDir(const char * name){
  //
  // Check if it is directory
  // 
  void *dir = gSystem->OpenDirectory(name);
  return (dir!=0);
}


Bool_t AlienToolkit::IsFile(const  char *name ){
  //
  // Check if is it file
  //
  Long_t id, size, flags, modtime ;
  Int_t ret = gSystem->GetPathInfo(name,&id, &size, &flags, &modtime);
  if (ret==0) return kFALSE;
  if ((flags&2)>0) return kFALSE;
  return kTRUE;
}


TGridCollection* AlienToolkit::MakeCollection(const char *path,  char *mask){
  //
  /*char *mask = "root_archive.zip";
    char *path = "/alice/cern.ch/user/m/miranov/test2007/"
  */
  //
  TGridResult* query = gGrid->Query(path,mask);  
  TGridCollection* collection=gGrid->OpenCollectionQuery(query);
  //  collection->SelectFile(mask,1,20); // select files 0-100
  collection->LookupSUrls();
  //collection->CheckIfOnline();
  collection->Reset();
  fCollection = collection;
  //
  collection->Reset();
  Int_t counter=0;
  Int_t counterm=0;
  Int_t counterf=0;
  TMap *filemap;
  while ( (filemap = collection->Next())) { 
    TIterator *nextfile = filemap->MakeIterator();
    TMap *attributes;
    while ((attributes = (TMap *) nextfile->Next())) {
      printf("%d\t%d\t%d\n",counter, counterm,counterf); 
      TMap * map = new TMap;
      fInfoArray.AddLast(map);

      TObjString *surl = new TObjString(collection->GetSURL(attributes->GetName()));
      TObjString *turl =  new TObjString(collection->GetTURL(attributes->GetName()));
      TObjString *lfn =  new TObjString(collection->GetLFN(attributes->GetName()));
      map->Add(new TObjString("alienLFN"),lfn);
      map->Add(new TObjString("alienSURL"),surl);
      map->Add(new TObjString("alienTURL"),turl);
     

      Int_t   isOnline = collection->IsOnline(attributes->GetName());
      printf("Base Name:\t%s\n",attributes->GetName());
      printf("Size:\t%d\n",(Int_t)collection->GetSize());
      printf("IsOnline:\t%d\n",collection->IsOnline());
      printf("IsSelected:\t%d\n",collection->IsSelected());
      printf("IsOnline:\t%d\n",isOnline);
      printf("LFN  Name:\t%s\n",lfn->String().Data());
      printf("TURL Name:\t%s\n",turl->String().Data());
      printf("SURL Name:\t%s\n",surl->String().Data());
      counter++;
      counterf++;
      
    }
    counterf=0;
    counterm++;
  }
}


void AlienToolkit::Stage(){
  //
  // Stage selected alien files
  //
  Int_t entries = fInfoArray.GetEntries();
  ofstream aout("stage.txt");
  for (Int_t i=0; i<entries;i++){
    TMap &map = *((TMap*)fInfoArray.At(i));
    TObjString *lfn = (TObjString*)map("alienLFN");
    if (!lfn) continue;
    printf("Staging submitfor\t%s\n",lfn->String().Data());
    aout<<"stage "<<lfn->String().Data()<<endl;
  }
  aout.close();
  gSystem->Exec("aliensh file:stage.txt");
}

void AlienToolkit::LocalCopy(const char *destination){
  //
  // Copy selected files to the destination directory
  // the LFN path name translated to the directory name replacing
  // separtor - the flat structure is created 

  Int_t entries = fInfoArray.GetEntries();
  ofstream aout("stage.txt");
  for (Int_t i=0; i<entries;i++){
    TMap &map = *((TMap*)fInfoArray.At(i));
    TObjString *lfn = (TObjString*)map("alienLFN");
    if (!lfn) continue;
    printf("Staging submitfor\t%s\n",lfn->String().Data());
    TString dnames=lfn->String().Data();
    dnames.ReplaceAll(".root_dir","");
    dnames.ReplaceAll("/","_");
    TString dname=destination;
    dname+=dnames;
    aout<<"cp  "<<lfn->String().Data()<<" "<<dname.Data()<<endl;
  }
  aout.close();
  gSystem->Exec("aliensh file:stage.txt");
}


void              AlienToolkit::PrintPFN(){
  //
  //
  // 
  Int_t entries = fInfoArray.GetEntries();
  for (Int_t i=0; i<entries;i++){
    TMap &map = *((TMap*)fInfoArray.At(i));
    TObjString *lfn = (TObjString*)map("alienLFN");
    TObjString *pfn = (TObjString*)map("alienSURL");
    if (!lfn) continue;
    if (!pfn) continue;
    printf("%s\n",pfn->String().Data());
  }
}

void   AlienToolkit::MakeJobList(const  char * outname, const char *outputPrefix, const char *action, const char *suffix){
  //
  //
  //
   Int_t entries = fInfoArray.GetEntries();
  ofstream aout(outname);
  for (Int_t i=0; i<entries;i++){
    TMap &map = *((TMap*)fInfoArray.At(i));
    TObjString *lfn = (TObjString*)map("alienLFN");
    TObjString *pfn = (TObjString*)map("alienSURL");
    if (!lfn) continue;
    if (!pfn) continue;
     if (lfn->String().Contains(".tag.")) continue;
    printf("Job info\t%s\n",lfn->String().Data());
    TString jobID=lfn->String().Data();
    jobID.ReplaceAll("/","_");
    //
    //
    //
    //
    TString outputDir=outputPrefix;
    outputDir+=lfn->String().Data();
    
    aout<<jobID<<" "<<pfn->String().Data()<<"  "<<outputDir.Data()<<" "<<action;
    if (suffix)  aout<<" "<<suffix<<"\n";
    aout<<endl;
  }
  aout.close();
  //
}


void AlienToolkit::RemoteCopy(const char *destination,Int_t maxfiles){
  //
  // Copy selected files to the destination directory
  // the LFN path name translated to the directory name replacing
  // separtor - the flat structure is created 

  Int_t entries = fInfoArray.GetEntries();
  ofstream *aout=0;
  for (Int_t i=0; i<entries;i++){
    if (i%maxfiles==0){
      if (aout) aout->close();
      aout = new ofstream(Form("stage_%d.sh",i/maxfiles));
      (*aout)<<"!/bin/bash\n";
      (*aout)<<"source ~/.balice\n";
    }
    TMap &map = *((TMap*)fInfoArray.At(i));
    TObjString *pfn = (TObjString*)map("alienSURL");
    TObjString *lfn = (TObjString*)map("alienLFN");
    if (!pfn) continue;
    if (!lfn) continue;
    TString dnames=lfn->String().Data();
    TString dname=destination;
    dname+=dnames;
    (*aout)<<"xrdcp -d 1 "<<pfn->String().Data()<<" "<<dname.Data()<<endl;
  }
  aout->close();
  //gSystem->Exec("aliensh file:stage.txt");
}

Bool_t AlienToolkit::ResubmitJobs(){
  //
  // Resubmit the processes finished in error state
  //
  //gSystem->Exec("alien_ps | grep EE > ps.txt")
  gSystem->Exec("alien_ps | grep EE |gawk '{print $2}' > psEE.txt");
  gSystem->Exec("alien_ps | grep EXPIRED |gawk '{print $2}' > psEXPIRED.txt");
  //
  ifstream in;
  in.open("psEXPIRED.txt");
  TString line;
  TString result;
  while(in.good()) {
    in >> line;
    //    line.ReplaceAll("-","");
    printf("Resubmiting %s\n",line.Data());
    Int_t pos = line.First("-");
    printf("%s\n",Form("alien_resubmit %s",&line.Data()[pos+1]));
    gSystem->Exec(Form("alien_resubmit %s",&line.Data()[pos+1]));
  }
  in.close();
}


/*
  Problems:
1. ????  fCollection.IsOnline("")

  #12 0x04e9a03f in XrdClientAdmin::IsFileOnline (this=0x94a91b0, vs=@0xbfe07f60, vb=@0xbfe07f80) at ../../src/XrdClient/XrdClientVector.hh:55
#13 0x0423a975 in TXNetSystem::IsOnline (this=0x9652800,
    path=0x9661488 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91")
    at netx/src/TXNetSystem.cxx:476
#14 0x04237c6e in TXNetFileStager::IsStaged (this=0x96526b8,
    path=0x9661400 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91")
    at netx/src/TXNetFileStager.cxx:63
#15 0x00551117 in TFileStager::GetStaged (this=0x96526b8, pathlist=0x9649820) at net/src/TFileStager.cxx:58
#16 0x010e5d96 in TAlienCollection::CheckIfOnline (this=0x95da8d8, bulk=true) at alien/src/TAlienCollection.cxx:1143


2. id xrootd on voalice04.cern.ch ?
   "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91"

3. How to check if the file exist in redirector ?
   
4. Ho

*/
Commit	Line	Data
7332008b	1	#include "TGrid.h"
	2	#include "TGridCollection.h"
	3	#include "TSystem.h"
	4	#include "TString.h"
	5	#include "TFile.h"
	6	#include "TMap.h"
	7	#include "TGridResult.h"
	8	#include "TAlien.h"
	9	#include "TAlienResult.h"
	10	#include <fstream>
	11
	12
	13
	14
	15
	16	/*
	17	gSystem->Load("libXrdClient.so");
	18	gSystem->Load("libNetx.so");
	19	.L $ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx+
	20	//Reconstructed data example
	21	char mask = "v14";
	22	char *path = "/alice/cern.ch/user/m/miranov/test2007/"
	23	// raw data
027ef365	24	char mask = "0000191RecPo*.root";
027ef365	25	char *path = "/alice/data/2008/LHC08a_TPC/"
7332008b	26
	27	AlienToolkit toolkit;
	28	toolkit.MakeCollection(path,mask);
	29
	30	//
027ef365	31	toolkit.MakeJobList("job.list","", "", "rec");
7332008b	32
	33	*/
	34
	35
	36
	37	class AlienToolkit: public TObject{
	38	public:
	39	AlienToolkit();
	40	TGridCollection* MakeCollection(const char path, char mask);
	41	void Stage();
	42	void LocalCopy(const char* destination);
	43	void RemoteCopy(const char* destination="root://gsiaf.gsi.de:1094/", Int_t maxfiles=20);
027ef365	44	void PrintPFN();
7332008b	45	void MakeJobList(const char * outname, const char outputPrefix, const char action, const char *suffix);
	46	static Bool_t IsDir(const char * name);
	47	static Bool_t IsFile(const char * name);
	48	static Bool_t ResubmitJobs();
027ef365	49
7332008b	50	public:
	51	TGridCollection *fCollection;
	52	TObjArray fInfoArray;
	53	ClassDef(AlienToolkit,1)
	54	};
	55
	56	ClassImp(AlienToolkit)
	57
	58	AlienToolkit::AlienToolkit():
	59	fCollection(0)
	60	{
	61	if (!gGrid)
	62	TGrid::Connect("alien://",0,0,"t");
	63	}
	64
	65
	66
	67
	68	Bool_t AlienToolkit::IsDir(const char * name){
	69	//
	70	// Check if it is directory
	71	//
	72	void *dir = gSystem->OpenDirectory(name);
	73	return (dir!=0);
	74	}
	75
	76
	77	Bool_t AlienToolkit::IsFile(const char *name ){
	78	//
	79	// Check if is it file
	80	//
	81	Long_t id, size, flags, modtime ;
	82	Int_t ret = gSystem->GetPathInfo(name,&id, &size, &flags, &modtime);
	83	if (ret==0) return kFALSE;
	84	if ((flags&2)>0) return kFALSE;
	85	return kTRUE;
	86	}
	87
	88
	89	TGridCollection* AlienToolkit::MakeCollection(const char path, char mask){
	90	//
	91	/char mask = "root_archive.zip";
	92	char *path = "/alice/cern.ch/user/m/miranov/test2007/"
	93	*/
	94	//
	95	TGridResult* query = gGrid->Query(path,mask);
	96	TGridCollection* collection=gGrid->OpenCollectionQuery(query);
	97	// collection->SelectFile(mask,1,20); // select files 0-100
	98	collection->LookupSUrls();
	99	//collection->CheckIfOnline();
	100	collection->Reset();
	101	fCollection = collection;
	102	//
	103	collection->Reset();
	104	Int_t counter=0;
	105	Int_t counterm=0;
	106	Int_t counterf=0;
	107	TMap *filemap;
	108	while ( (filemap = collection->Next())) {
	109	TIterator *nextfile = filemap->MakeIterator();
	110	TMap *attributes;
	111	while ((attributes = (TMap *) nextfile->Next())) {
	112	printf("%d\t%d\t%d\n",counter, counterm,counterf);
	113	TMap * map = new TMap;
114	fInfoArray.AddLast(map);
115
116	TObjString *surl = new TObjString(collection->GetSURL(attributes->GetName()));
117	TObjString *turl = new TObjString(collection->GetTURL(attributes->GetName()));
118	TObjString *lfn = new TObjString(collection->GetLFN(attributes->GetName()));
119	map->Add(new TObjString("alienLFN"),lfn);
120	map->Add(new TObjString("alienSURL"),surl);
121	map->Add(new TObjString("alienTURL"),turl);
122
123
124	Int_t isOnline = collection->IsOnline(attributes->GetName());
125	printf("Base Name:\t%s\n",attributes->GetName());
126	printf("Size:\t%d\n",(Int_t)collection->GetSize());
127	printf("IsOnline:\t%d\n",collection->IsOnline());
128	printf("IsSelected:\t%d\n",collection->IsSelected());
129	printf("IsOnline:\t%d\n",isOnline);
130	printf("LFN Name:\t%s\n",lfn->String().Data());
131	printf("TURL Name:\t%s\n",turl->String().Data());
132	printf("SURL Name:\t%s\n",surl->String().Data());
133	counter++;
134	counterf++;
135
136	}
137	counterf=0;
138	counterm++;
139	}
140	}
141
142
143	void AlienToolkit::Stage(){
144	//
145	// Stage selected alien files
146	//
147	Int_t entries = fInfoArray.GetEntries();
148	ofstream aout("stage.txt");
149	for (Int_t i=0; i<entries;i++){
150	TMap &map = ((TMap)fInfoArray.At(i));
151	TObjString lfn = (TObjString)map("alienLFN");
152	if (!lfn) continue;
153	printf("Staging submitfor\t%s\n",lfn->String().Data());
154	aout<<"stage "<<lfn->String().Data()<<endl;
155	}
156	aout.close();
157	gSystem->Exec("aliensh file:stage.txt");
158	}
159
160	void AlienToolkit::LocalCopy(const char *destination){
161	//
162	// Copy selected files to the destination directory
163	// the LFN path name translated to the directory name replacing
164	// separtor - the flat structure is created
165
166	Int_t entries = fInfoArray.GetEntries();
167	ofstream aout("stage.txt");
168	for (Int_t i=0; i<entries;i++){
169	TMap &map = ((TMap)fInfoArray.At(i));
170	TObjString lfn = (TObjString)map("alienLFN");
171	if (!lfn) continue;
172	printf("Staging submitfor\t%s\n",lfn->String().Data());
173	TString dnames=lfn->String().Data();
174	dnames.ReplaceAll(".root_dir","");
175	dnames.ReplaceAll("/","_");
176	TString dname=destination;
177	dname+=dnames;
178	aout<<"cp "<<lfn->String().Data()<<" "<<dname.Data()<<endl;
179	}
180	aout.close();
181	gSystem->Exec("aliensh file:stage.txt");
182	}
183
184
027ef365	185	void AlienToolkit::PrintPFN(){
	186	//
	187	//
	188	//
	189	Int_t entries = fInfoArray.GetEntries();
	190	for (Int_t i=0; i<entries;i++){
	191	TMap &map = ((TMap)fInfoArray.At(i));
	192	TObjString lfn = (TObjString)map("alienLFN");
	193	TObjString pfn = (TObjString)map("alienSURL");
	194	if (!lfn) continue;
	195	if (!pfn) continue;
	196	printf("%s\n",pfn->String().Data());
	197	}
	198	}
	199
7332008b	200	void AlienToolkit::MakeJobList(const char * outname, const char outputPrefix, const char action, const char *suffix){
	201	//
	202	//
	203	//
	204	Int_t entries = fInfoArray.GetEntries();
	205	ofstream aout(outname);
	206	for (Int_t i=0; i<entries;i++){
	207	TMap &map = ((TMap)fInfoArray.At(i));
	208	TObjString lfn = (TObjString)map("alienLFN");
	209	TObjString pfn = (TObjString)map("alienSURL");
	210	if (!lfn) continue;
	211	if (!pfn) continue;
9af56b10	212	if (lfn->String().Contains(".tag.")) continue;
7332008b	213	printf("Job info\t%s\n",lfn->String().Data());
	214	TString jobID=lfn->String().Data();
	215	jobID.ReplaceAll("/","_");
	216	//
	217	//
027ef365	218	//
027ef365	219	//
7332008b	220	TString outputDir=outputPrefix;
	221	outputDir+=lfn->String().Data();
	222
497c5329	223	aout<<jobID<<" "<<pfn->String().Data()<<" "<<outputDir.Data()<<" "<<action;
497c5329	224	if (suffix) aout<<" "<<suffix<<"\n";
7332008b	225	aout<<endl;
	226	}
	227	aout.close();
	228	//
	229	}
	230
	231
	232	void AlienToolkit::RemoteCopy(const char *destination,Int_t maxfiles){
	233	//
	234	// Copy selected files to the destination directory
	235	// the LFN path name translated to the directory name replacing
	236	// separtor - the flat structure is created
	237
	238	Int_t entries = fInfoArray.GetEntries();
	239	ofstream *aout=0;
	240	for (Int_t i=0; i<entries;i++){
	241	if (i%maxfiles==0){
	242	if (aout) aout->close();
	243	aout = new ofstream(Form("stage_%d.sh",i/maxfiles));
	244	(*aout)<<"!/bin/bash\n";
	245	(*aout)<<"source ~/.balice\n";
	246	}
	247	TMap &map = ((TMap)fInfoArray.At(i));
	248	TObjString pfn = (TObjString)map("alienSURL");
	249	TObjString lfn = (TObjString)map("alienLFN");
	250	if (!pfn) continue;
	251	if (!lfn) continue;
	252	TString dnames=lfn->String().Data();
	253	TString dname=destination;
	254	dname+=dnames;
	255	(*aout)<<"xrdcp -d 1 "<<pfn->String().Data()<<" "<<dname.Data()<<endl;
	256	}
	257	aout->close();
	258	//gSystem->Exec("aliensh file:stage.txt");
	259	}
	260
	261	Bool_t AlienToolkit::ResubmitJobs(){
	262	//
	263	// Resubmit the processes finished in error state
	264	//
	265	//gSystem->Exec("alien_ps \| grep EE > ps.txt")
	266	gSystem->Exec("alien_ps \| grep EE \|gawk '{print $2}' > psEE.txt");
	267	gSystem->Exec("alien_ps \| grep EXPIRED \|gawk '{print $2}' > psEXPIRED.txt");
	268	//
	269	ifstream in;
	270	in.open("psEXPIRED.txt");
	271	TString line;
	272	TString result;
	273	while(in.good()) {
	274	in >> line;
	275	// line.ReplaceAll("-","");
	276	printf("Resubmiting %s\n",line.Data());
	277	Int_t pos = line.First("-");
	278	printf("%s\n",Form("alien_resubmit %s",&line.Data()[pos+1]));
	279	gSystem->Exec(Form("alien_resubmit %s",&line.Data()[pos+1]));
	280	}
	281	in.close();
	282	}
	283
	284
	285
	286
	287
	288	/*
289	Problems:
290	1. ???? fCollection.IsOnline("")
291
292	#12 0x04e9a03f in XrdClientAdmin::IsFileOnline (this=0x94a91b0, vs=@0xbfe07f60, vb=@0xbfe07f80) at ../../src/XrdClient/XrdClientVector.hh:55
293	#13 0x0423a975 in TXNetSystem::IsOnline (this=0x9652800,
294	path=0x9661488 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91")
295	at netx/src/TXNetSystem.cxx:476
296	#14 0x04237c6e in TXNetFileStager::IsStaged (this=0x96526b8,
297	path=0x9661400 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91")
298	at netx/src/TXNetFileStager.cxx:63
299	#15 0x00551117 in TFileStager::GetStaged (this=0x96526b8, pathlist=0x9649820) at net/src/TFileStager.cxx:58
300	#16 0x010e5d96 in TAlienCollection::CheckIfOnline (this=0x95da8d8, bulk=true) at alien/src/TAlienCollection.cxx:1143
301
302
303	2. id xrootd on voalice04.cern.ch ?
304	"root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91"
305
306	3. How to check if the file exist in redirector ?
307
308	4. Ho
309
310	*/
311