+++ /dev/null
-#include <fstream>
-#include <TFile.h>
-#include <TSystem.h>
-#include <TMap.h>
-
-/*
-
- Process all jobs specified in the job.list file
- Concurent agents acces the list
- Once starting to procces given job the jonb is locked
-
-*/
-
-
-
-/*
-
-
-.L $ALICE_ROOT/TPC/macros/testTPC/AliTPCjobs.cxx+
-
-
-AliTPCJobs jobs;
-jobs.fJobFile="job.list"
-jobs.ProcessAllJobs();
-*/
-
-
-class AliTPCJobs : public TNamed{
-public:
- AliTPCJobs();
- void GetListOfJobs();
-
- void ProcessAllJobs();
- Bool_t GetNextJob();
- Bool_t GetNextJobOld();
- void ProcessJob(TString jobID, TString inputData, TString outputDir, TString action);
-
- void SetLock(TString jobID);
- void SetDone(TString jobID);
- void SetFail(TString jobID);
-
- void Stage(TString inputData,TString jobID);
-
- Bool_t IsLocked(TString jobID);
- Bool_t IsFail(TString jobID);
- Bool_t IsStaged(TString inputData);
- Bool_t IsCastorFile(TString filename);
- //
- //
- TString fJobFile;
- TString fWorkDir;
- TObjArray fJobInfo; // array of job infos
-
- ClassDef(AliTPCJobs,0)
-};
-
- ClassImp(AliTPCJobs)
-
-AliTPCJobs::AliTPCJobs(){
- //
- // load neccessary libraries
- //
- gSystem->Load("libXrdClient.so");
- gSystem->Load("libNetx.so");
-}
-
-void AliTPCJobs::GetListOfJobs(){
- //
- // char * fJobFile ="job.list"
- //
- // Update the list of jobs - using the "local file"
- //
- ifstream ins;
- ins.open(fJobFile);
- char line[10000];
- TObjArray *arrayList=0;
- while(ins.good()) {
- ins.getline(line,10000);
- //printf("%s\n",line);
- arrayList = (TString(line)).Tokenize(" ");
- if (arrayList->GetEntries()>3){
- TObjString * jobID = (TObjString*)arrayList->At(0);
- TMap *map = (TMap*)fJobInfo.FindObject(jobID->GetName());
- if (!map) {
- map = new TMap();
- map->SetName(jobID->GetName());
- fJobInfo.AddLast(map);
- map->Add(new TObjString("Input"),arrayList->At(1));
- map->Add(new TObjString("Output"),arrayList->At(2));
- map->Add(new TObjString("Action"),arrayList->At(3));
- map->Add(new TObjString("Status"),new TObjString("0"));
- }
- }else{
- delete arrayList;
- }
- }
-}
-
-
-void AliTPCJobs::ProcessAllJobs(){
- //
- //
- //
- GetListOfJobs();
- Int_t counter=0;
- while (GetNextJob()){
- //
- printf("PROCESSING JOB\t%d\n",counter);
- counter++;
- if (!GetNextJob()) break;
- }
-}
-
-
-Bool_t AliTPCJobs::GetNextJob(){
- //
- // GetNextJob - get job from the list which is not locked
- //
- TString id;
- TString inputData;
- TString outputDir;
- TString action;
- TString status;
-
- Bool_t hasJob=kFALSE;
- Int_t entries = fJobInfo.GetEntries();
- TMap *map=0;
- for (Int_t i=0; i<entries; i++){
- map = (TMap*)fJobInfo.At(i);
- status=(*map)("Status")->GetName();
- id=map->GetName();
- inputData=(*map)("Input")->GetName();
- outputDir=(*map)("Output")->GetName();
- action=(*map)("Action")->GetName();
- if (status.Contains("1")) continue;
-// if (IsFail(id)) continue;
- if (!IsLocked(id)){
- hasJob=kTRUE;
- ((TObjString*)(*map)("Status"))->SetString("1");
- if (!IsStaged(inputData)){
- //Stage(inputData,id);
- continue;
- }
- break;
- }
- }
-
- printf("Process %s\n",id.Data());
- if (hasJob) {
- ProcessJob(id,inputData,outputDir, action);
- }
- return hasJob;
-}
-
-
-
-Bool_t AliTPCJobs::GetNextJobOld(){
- //
- // GetNextJob - get job from the list which is not locked
- //
- ifstream ins;
- ins.open(fJobFile);
- TString id;
- TString inputData;
- TString outputDir;
- TString action;
- Bool_t hasJob=kFALSE;
- while(ins.good()) {
- ins>>id;
- ins>>inputData;
- ins>>outputDir;
- ins>>action;
- //if (!inputData.Contains(".root")) SetFail(id);
- if (IsFail(id)) continue;
- if (!IsLocked(id)){
- hasJob=kTRUE;
- break;
- }
- if (!IsStaged(inputData)){
- //Stage(inputData,id);
- continue;
- }
- }
- printf("Process %s\n",id.Data());
- if (hasJob) ProcessJob(id,inputData,outputDir, action);
- return hasJob;
-}
-
-
-void AliTPCJobs::SetLock(TString jobID){
- //
- //
- //
- printf("touch out/%s.lock\n",jobID.Data());
- gSystem->Exec(Form("touch out/%s.lock",jobID.Data()));
-}
-
-void AliTPCJobs::SetDone(TString jobID){
- //
- //
- //
- printf("touch out/%s.done\n",jobID.Data());
- gSystem->Exec(Form("touch out/%s.done", jobID.Data()));
-}
-
-void AliTPCJobs::SetFail(TString jobID){
- //
- //
- //
- printf("touch out/%s.fail\n",jobID.Data());
- gSystem->Exec(Form("touch out/%s.fail", jobID.Data()));
-}
-
-void AliTPCJobs::Stage(TString inputData, TString jobID){
- //
- // stage file
- //
- inputData.ReplaceAll("root://voalice04.cern.ch:1094/","");
- if ( !IsCastorFile(inputData) ) return;
- char command[1000];
- sprintf(command,"stager_get -M %s \| grep SUBREQUEST_FAILED ",inputData.Data());
- FILE *pipe = gSystem->OpenPipe(command,"r");
- TString result;
- result.Gets(pipe);
- gSystem->ClosePipe(pipe);
- if ( result.Contains("SUBREQUEST_FAILED") ){
- SetFail(jobID);
- }
-}
-
-Bool_t AliTPCJobs::IsLocked(TString jobID){
- TString path = "out/";
- path+=jobID;
- path+=".lock";
- Long_t pid; Long_t psize; Long_t pflags; Long_t pmodtime;
- Int_t status = gSystem->GetPathInfo(path,&pid,&psize,&pflags,&pmodtime);
- return (status==0);
-}
-
-Bool_t AliTPCJobs::IsFail(TString jobID){
- //
- //
- //
- TString path = "out/";
- path+=jobID;
- path+=".fail";
- Long_t pid; Long_t psize; Long_t pflags; Long_t pmodtime;
- Int_t status = gSystem->GetPathInfo(path,&pid,&psize,&pflags,&pmodtime);
- return (status==0);
-}
-
-Bool_t AliTPCJobs::IsStaged(TString inputData){
- //
- // check if file bname is staged
- //
- inputData.ReplaceAll("root://voalice04.cern.ch:1094/","");
- if ( !IsCastorFile(inputData) ) return kTRUE;
- char command[1000];
- sprintf(command,"stager_qry -M %s \| grep /castor \| gawk \'{ print $3;}\'",inputData.Data());
- FILE *pipe = gSystem->OpenPipe(command,"r");
- TString result;
- result.Gets(pipe);
- gSystem->ClosePipe(pipe);
- if ( result.Contains("STAGED") ) return kTRUE;
-
- return kFALSE;
-}
-
-Bool_t AliTPCJobs::IsCastorFile(TString filename){
- //
- // check if filename begins with 'castor'
- //
- return filename.BeginsWith("/castor/");
-}
-
-
-void AliTPCJobs::ProcessJob(TString jobID, TString inputData, TString outputDir, TString action){
- //
- //
- // 1. Create lock file
- // 2. Get Input data
- // 3. Process data
- // 4. Create Done file
- SetLock(jobID);
- if (action.Contains("TEST")){
- char command[10000];
- sprintf(command,"$ALICE_ROOT/TPC/macros/testTPC/action.sh %s %s %s %s", jobID.Data(), inputData.Data(), outputDir.Data(), action.Data());
- printf("%s\n\n",command);
- SetDone(jobID);
- return;
- }
- if (action.Contains("COPY")){
-
- char command[10000];
- sprintf(command,"mkdirhier %s",gSystem->DirName(outputDir.Data()));
- printf("Exec\t%s\n", command);
- gSystem->Exec(command);
- sprintf(command,"xrdcp -DIFirstConnectMaxCnt 4 -DIConnectTimeout 4 -DIRequestTimeout 4 -DIMaxRedirectcount 4 -DIRedirCntTimeout 4 %s\t%s\n",inputData.Data(), outputDir.Data());
- printf("Exec\t%s\n", command);
- gSystem->Exec(command);
- //TFile::Cp(inputData.Data(), outputDir.Data());
- }else{
- char command[10000];
- sprintf(command,"$ALICE_ROOT/TPC/macros/testTPC/action.sh %s %s %s %s", jobID.Data(), inputData.Data(), outputDir.Data(), action.Data());
- printf("%s\n\n",command);
- gSystem->Exec(command);
- printf("\n\n");
-
- // create temp work dir
-// TString processDir=fWorkDir+jobID+action;
-// Int_t res = gSystem->mkdir(processDir,kTRUE);
-// if ( res==-1 ){
-// AliWarning(Form("Cannot create dir/already exists: '%s'",processDir.Data()));
-// return;
-// }
- }
- SetDone(jobID);
-}
-
-
-
-void AliTPCjobs(){
- //
- //
- //
- AliTPCJobs jobs;
- jobs.fJobFile="job.list";
- jobs.ProcessAllJobs();
-}
+++ /dev/null
-#include "TGrid.h"
-#include "TGridCollection.h"
-#include "TSystem.h"
-#include "TString.h"
-#include "TFile.h"
-#include "TMap.h"
-#include "TGridResult.h"
-#include "TAlien.h"
-#include "TAlienResult.h"
-#include <fstream>
-
-/*
- Simple toolkit for Alien
- Use ROOT C++ interfaces classes to access information from ALIEN
-
- Important functionality:
- MakeJobList - prepared the list which is later used by agent.sh
- to analyze data on batch system
-
-
-
- See example bellow and class heeader for details.
- !!! The functionality not expelicitly mentioned in example can be not working
- Code is still under development
-
-*/
-
-
-
-/*
- gSystem->Load("libXrdClient.so");
- gSystem->Load("libNetx.so");
- //Raw data example
- char *mask = "20225";
- char *path = "/alice/data/2008/LHC08d"
-
- .L $ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx+
- //
- AlienToolkit toolkit;
- toolkit.MakeCollection(path,mask); // make a list of the registerd data
- //toolkit.StageCastor(); // stage files on castor
- //
- toolkit.MakeJobList("job.list","", "", "FindKrClustersRaw");
-
-*/
-
-
-
-class AlienToolkit: public TObject{
-public:
- AlienToolkit();
- TGridCollection* MakeCollection(const char *path, char *mask);
- void Stage();
- void StageCastor();
- void LocalCopy(const char* destination);
- void RemoteCopy(const char* destination="root://gsiaf.gsi.de:1094/", Int_t maxfiles=20);
- void RemoteCopyAlien(const char* destination="root://gsiaf.gsi.de:1094/", Int_t maxfiles=20);
-
- void PrintPFN();
- void PrintLFN();
- void MakeJobList(const char * outname, const char *outputPrefix, const char *action, const char *suffix);
- static Bool_t IsDir(const char * name);
- static Bool_t IsFile(const char * name);
- static Bool_t ResubmitJobs();
-
-public:
- TGridCollection *fCollection;
- TObjArray fInfoArray;
- ClassDef(AlienToolkit,1)
-};
-
-ClassImp(AlienToolkit)
-
-AlienToolkit::AlienToolkit():
- fCollection(0)
-{
- if (!gGrid)
- TGrid::Connect("alien://",0,0,"t");
-}
-
-
-
-
-Bool_t AlienToolkit::IsDir(const char * name){
- //
- // Check if it is directory
- //
- void *dir = gSystem->OpenDirectory(name);
- return (dir!=0);
-}
-
-
-Bool_t AlienToolkit::IsFile(const char *name ){
- //
- // Check if is it file
- //
- Long_t id, size, flags, modtime ;
- Int_t ret = gSystem->GetPathInfo(name,&id, &size, &flags, &modtime);
- if (ret==0) return kFALSE;
- if ((flags&2)>0) return kFALSE;
- return kTRUE;
-}
-
-
-TGridCollection* AlienToolkit::MakeCollection(const char *path, char *mask){
- //
- /*char *mask = "root_archive.zip";
- char *path = "/alice/cern.ch/user/m/miranov/test2007/"
- */
- //
- TGridResult* query = gGrid->Query(path,mask);
- TGridCollection* collection=gGrid->OpenCollectionQuery(query);
- // collection->SelectFile(mask,1,20); // select files 0-100
- collection->LookupSUrls();
- //collection->CheckIfOnline();
- collection->Reset();
- fCollection = collection;
- //
- collection->Reset();
- Int_t counter=0;
- Int_t counterm=0;
- Int_t counterf=0;
- TMap *filemap;
- TObjString lastLFN;
-
- while ( (filemap = collection->Next())) {
- TIterator *nextfile = filemap->MakeIterator();
- TMap *attributes;
- while ((attributes = (TMap *) nextfile->Next())) {
- printf("%d\t%d\t%d\n",counter, counterm,counterf);
- TMap * map = new TMap;
-
- TObjString *surl = new TObjString(collection->GetSURL(attributes->GetName()));
- TObjString *turl = new TObjString(collection->GetTURL(attributes->GetName()));
- TObjString *lfn = new TObjString(collection->GetLFN(attributes->GetName()));
- map->Add(new TObjString("alienLFN"),lfn);
- map->Add(new TObjString("alienSURL"),surl);
- map->Add(new TObjString("alienTURL"),turl);
-
- if (lastLFN.String().CompareTo(lfn->String())==0) continue;
- lastLFN = *lfn;
- Int_t isOnline = collection->IsOnline(attributes->GetName());
- printf("Base Name:\t%s\n",attributes->GetName());
- printf("Size:\t%d\n",(Int_t)collection->GetSize());
- printf("IsOnline:\t%d\n",collection->IsOnline());
- printf("IsSelected:\t%d\n",collection->IsSelected());
- printf("IsOnline:\t%d\n",isOnline);
- printf("LFN Name:\t%s\n",lfn->String().Data());
- printf("TURL Name:\t%s\n",turl->String().Data());
- printf("SURL Name:\t%s\n",surl->String().Data());
- counter++;
- counterf++;
- fInfoArray.AddLast(map);
-
- }
- counterf=0;
- counterm++;
- }
-}
-
-
-void AlienToolkit::Stage(){
- //
- // Stage selected alien files
- //
- Int_t entries = fInfoArray.GetEntries();
- ofstream aout("stage.txt");
- for (Int_t i=0; i<entries;i++){
- TMap &map = *((TMap*)fInfoArray.At(i));
- TObjString *lfn = (TObjString*)map("alienLFN");
- if (!lfn) continue;
- printf("Staging submitfor\t%s\n",lfn->String().Data());
- aout<<"stage "<<lfn->String().Data()<<endl;
- }
- aout.close();
- gSystem->Exec("aliensh file:stage.txt");
-}
-
-void AlienToolkit::StageCastor(){
- //
- // Stage selected alien files
- //
- Int_t entries = fInfoArray.GetEntries();
- ofstream aout("stage.sh");
- aout<<"#!/usr/local/bin/bash"<<endl;
- for (Int_t i=0; i<entries;i++){
- TMap &map = *((TMap*)fInfoArray.At(i));
- TObjString *pfn = (TObjString*)map("alienSURL");
- if (!pfn) continue;
- if (!pfn->String().Contains("//castor")) continue;
- Char_t * cstr = &( pfn->String()[pfn->String().Index("/castor")]);
- if (!cstr) continue;
- printf("Staging submitfor\t%s\n",cstr);
- char command[1000];
- sprintf(command,"stager_qry -M %s | grep /castor | gawk \'{print $3;}\'", cstr);
- gSystem->Exec(command);
- sprintf(command,"stager_get -M %s", cstr);
- aout<<command<<endl;
- //gSystem->Exec(command);
- }
- aout.close();
- //gSystem->Exec("source stage.sh &");
-}
-
-void AlienToolkit::LocalCopy(const char *destination){
- //
- // Copy selected files to the destination directory
- // the LFN path name translated to the directory name replacing
- // separtor - the flat structure is created
-
- Int_t entries = fInfoArray.GetEntries();
- ofstream aout("stage.txt");
- for (Int_t i=0; i<entries;i++){
- TMap &map = *((TMap*)fInfoArray.At(i));
- TObjString *lfn = (TObjString*)map("alienLFN");
- if (!lfn) continue;
- printf("Staging submitfor\t%s\n",lfn->String().Data());
- TString dnames=lfn->String().Data();
- dnames.ReplaceAll(".root_dir","");
- dnames.ReplaceAll("/","_");
- TString dname=destination;
- dname+=dnames;
- aout<<"cp "<<lfn->String().Data()<<" "<<dname.Data()<<endl;
- }
- aout.close();
- gSystem->Exec("aliensh file:stage.txt");
-}
-
-
-void AlienToolkit::PrintPFN(){
- //
- //
- //
- Int_t entries = fInfoArray.GetEntries();
- for (Int_t i=0; i<entries;i++){
- TMap &map = *((TMap*)fInfoArray.At(i));
- TObjString *lfn = (TObjString*)map("alienLFN");
- TObjString *pfn = (TObjString*)map("alienSURL");
- if (!lfn) continue;
- if (!pfn) continue;
- printf("%s\n",pfn->String().Data());
- }
-}
-
-void AlienToolkit::PrintLFN(){
- //
- //
- //
- Int_t entries = fInfoArray.GetEntries();
- for (Int_t i=0; i<entries;i++){
- TMap &map = *((TMap*)fInfoArray.At(i));
- TObjString *lfn = (TObjString*)map("alienLFN");
- TObjString *pfn = (TObjString*)map("alienSURL");
- if (!lfn) continue;
- if (!pfn) continue;
- printf("%s\n",lfn->String().Data());
- }
-}
-
-
-void AlienToolkit::MakeJobList(const char * outname, const char *outputPrefix, const char *action, const char *suffix){
- //
- //
- //
- Int_t entries = fInfoArray.GetEntries();
- ofstream aout(outname);
- for (Int_t i=0; i<entries;i++){
- TMap &map = *((TMap*)fInfoArray.At(i));
- TObjString *lfn = (TObjString*)map("alienLFN");
- TObjString *pfn = (TObjString*)map("alienSURL");
- if (!lfn) continue;
- if (!pfn) continue;
- if (lfn->String().Contains(".tag.")) continue;
- printf("Job info\t%s\n",lfn->String().Data());
- TString jobID=lfn->String().Data();
- jobID.ReplaceAll("/","_");
- //
- //
- //
- //
- TString outputDir=outputPrefix;
- outputDir+=lfn->String().Data();
-
- aout<<jobID<<" "<<pfn->String().Data()<<" "<<outputDir.Data()<<" "<<action;
- if (suffix) aout<<" "<<suffix<<"\n";
- aout<<endl;
- }
- aout.close();
- //
-}
-
-
-void AlienToolkit::RemoteCopy(const char *destination,Int_t maxfiles){
- //
- // Copy selected files to the destination directory
- // the LFN path name translated to the directory name replacing
- // separtor - the flat structure is created
-
- Int_t entries = fInfoArray.GetEntries();
- ofstream *aout=0;
- for (Int_t i=0; i<entries;i++){
- if (i%maxfiles==0){
- if (aout) aout->close();
- aout = new ofstream(Form("stage_%d.sh",i/maxfiles));
- (*aout)<<"!/bin/bash\n";
- (*aout)<<"source ~/.balice\n";
- }
- TMap &map = *((TMap*)fInfoArray.At(i));
- TObjString *pfn = (TObjString*)map("alienSURL");
- TObjString *lfn = (TObjString*)map("alienLFN");
- if (!pfn) continue;
- if (!lfn) continue;
- TString dnames=lfn->String().Data();
- TString dname=destination;
- dname+=dnames;
-
- (*aout)<< "mkdirhier "<<gSystem->DirName(dname.Data())<<endl;
- (*aout)<<"xrdcp -d 1 "<<pfn->String().Data()<<" "<<dname.Data()<<endl;
- if (dnames.Contains(".zip")){
- (*aout)<<"unzip "<<pfn->String().Data()<<endl;
- }
- }
- aout->close();
- //gSystem->Exec("aliensh file:stage.txt");
-}
-
-void AlienToolkit::RemoteCopyAlien(const char *destination,Int_t maxfiles){
- //
- // Copy selected files to the destination directory
- // the LFN path name translated to the directory name replacing
- // separtor - the flat structure is created
-
- Int_t entries = fInfoArray.GetEntries();
- ofstream *aout=0;
- for (Int_t i=0; i<entries;i++){
- if (i%maxfiles==0){
- if (aout) aout->close();
- aout = new ofstream(Form("stage_%d.sh",i/maxfiles));
- (*aout)<<"!/bin/bash\n";
- (*aout)<<"source ~/.balice\n";
- (*aout)<<"source ~/.aliensetup\n";
- }
- TMap &map = *((TMap*)fInfoArray.At(i));
- TObjString *pfn = (TObjString*)map("alienSURL");
- TObjString *lfn = (TObjString*)map("alienLFN");
- if (!pfn) continue;
- if (!lfn) continue;
- TString dnames=lfn->String().Data();
- TString dname=destination;
- dname+=dnames;
- (*aout)<< " echo Copy"<<gSystem->DirName(dname.Data())<<endl;
- (*aout)<< "mkdirhier "<<gSystem->DirName(dname.Data())<<endl;
- (*aout)<<"alien_cp alien://"<<lfn->String().Data()<<" "<<dname.Data()<<endl;
- if (dnames.Contains(".zip")){
- (*aout)<< "cd "<<gSystem->DirName(dname.Data())<<endl;
- (*aout)<<"unzip "<<gSystem->BaseName(dname.Data())<<endl;
- }
- }
- aout->close();
- //gSystem->Exec("aliensh file:stage.txt");
-}
-
-Bool_t AlienToolkit::ResubmitJobs(){
- //
- // Resubmit the processes finished in error state
- //
- //gSystem->Exec("alien_ps | grep EE > ps.txt")
- gSystem->Exec("alien_ps | grep EE |gawk '{print $2}' > psEE.txt");
- gSystem->Exec("alien_ps | grep EXPIRED |gawk '{print $2}' > psEXPIRED.txt");
- //
- ifstream in;
- in.open("psEXPIRED.txt");
- TString line;
- TString result;
- while(in.good()) {
- in >> line;
- // line.ReplaceAll("-","");
- printf("Resubmiting %s\n",line.Data());
- Int_t pos = line.First("-");
- printf("%s\n",Form("alien_resubmit %s",&line.Data()[pos+1]));
- gSystem->Exec(Form("alien_resubmit %s",&line.Data()[pos+1]));
- }
- in.close();
-}
-
-
-
-
-
-/*
- Problems:
-1. ???? fCollection.IsOnline("")
-
- #12 0x04e9a03f in XrdClientAdmin::IsFileOnline (this=0x94a91b0, vs=@0xbfe07f60, vb=@0xbfe07f80) at ../../src/XrdClient/XrdClientVector.hh:55
-#13 0x0423a975 in TXNetSystem::IsOnline (this=0x9652800,
- path=0x9661488 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91")
- at netx/src/TXNetSystem.cxx:476
-#14 0x04237c6e in TXNetFileStager::IsStaged (this=0x96526b8,
- path=0x9661400 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91")
- at netx/src/TXNetFileStager.cxx:63
-#15 0x00551117 in TFileStager::GetStaged (this=0x96526b8, pathlist=0x9649820) at net/src/TFileStager.cxx:58
-#16 0x010e5d96 in TAlienCollection::CheckIfOnline (this=0x95da8d8, bulk=true) at alien/src/TAlienCollection.cxx:1143
-
-
-2. id xrootd on voalice04.cern.ch ?
- "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91"
-
-3. How to check if the file exist in redirector ?
-
-4. Ho
-
-*/
-
+++ /dev/null
-
-The "small" framework to make fast action(e.g reconstruction) pass using batch systems
-
-Example usage:
- $ALICE_ROOT/TPC/macros/testTPC/agent.sh /afs/cern.ch/user/m/miranov/public/test2008/reckr191xx
- if you run locally you should cd to your temp dir first, with 'cd /tmp/$USER'
-
- Or submit as many agents you want on batch machines:
- bsub -q 1nd $ALICE_ROOT/TPC/macros/testTPC/agent.sh /afs/cern.ch/user/m/miranov/public/test2008/reckr191xx
-
-
-AGENT to run some action:
-3 components to be present
-
-0. CREATE ACTION LIST
-
- 0.a Setup root, aliroot, alien (e.g. by doing 'source ~miranov/public/.bagentsetup')
- 0.a Get a valid token: with 'alien-token-init'
- 0.b source the alien env: with 'source /tmp/gclient_env_$UID'
- 0.c create the action list 'job.list' by following the
- example in '$ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx'
- 0.d if the first lines of this file contains entries like
- '/alice/data/2008/LHC08a_TPC/000028164/collection'
- they need to be deleted
-
-
-
-1. WORKING DIRECTORY SETUP (in prev. example /afs/cern.ch/user/m/miranov/public/test2008/reckr191xx),
- what needs to be present:
-
- 1.a the action list (called job.list - hardwired)
- 1.b create an empty directory called 'out'
- 1.c create a directory called 'macros', containing the '<action>.C' scripts,
- e.g 'rec.C' 'FindKrClustersRaw.C' etc.
-
-
-2. ENVIRONMENT SETUP:
-
- 2.a Copy the script to setup aliroot, root and the user specific env to the ~/.bagentsetup
- (see /afs/cern.ch/user/m/miranov/public/.bagentsetup for an example)
- 2.b Export envirnment variables needed OCDB_PATH, AGENTINPUTTYPE, AGENTSE in the .bagentsetup script
- 2.c Get a valid token: with 'alien-token-init' if not done above
- 2.d Copy certificates to default place - ~/.agentauth/
- cp /tmp/*$UID* ~/.agentauth/
- 2.e Run agent as described in the beginning examples
-
-
-
-
-
-REMOTE COPIES
-
-Following script genearte a bash script stage.sh
-with the xrdcp commands for selected files
-(see AlienToolkit.cxx - AlienToolkit::MakeCollection(dir, mask))
-
-Example:
-gSystem->Load("libXrdClient.so");
-gSystem->Load("libNetx.so");
-.L $ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx+
-//
-AlienToolkit toolkit;
-toolkit.MakeCollection("/alice/cern.ch/user/m/miranov/","08000031843019.1*AliESDs*")
-toolkit.RemoteCopy("/lustre_alpha/alice/TPCdata/cosmics_may2008/",10000000000)
+++ /dev/null
-#!/bin/bash
-#aguments
-#1 TString jobID,
-#2 TString inputData
-#3 TString outputDir
-#4 TString action
-
-echo INPUT ARGUMENT FOR ACTION
-echo $1
-echo $2
-echo $3
-echo $4
-#
-#
-#
-
-#chek if input file is a root file
-echo $2 | grep .root || exit
-
-olddir=`pwd`
-mkdir -p $jobhome/$1
-mkdir -p $jobhome/$1/$4
-cd $jobhome/$1/$4
-ln -sf $olddir home
-echo HOME DIR `pwd`
-#
-#
-#
-echo GETTING DATA - START
-date
-
-CISIZE=0
-if [ $AGENTINPUTTYPE -eq 2 ]; then
- CINAME=`echo $2| sed s_root://voalice04.cern.ch:1094/__`
- echo CASTOR COPY $CINAME ;
- CISIZE=`nsls -l $CINAME | gawk '{print $5}'`
- if [ $CISIZE -gt 1000000 ]; then
- rfcp $CINAME data.root
- else
- echo FILE TOO SMALL
- exit
- fi;
-fi;
-if [ $AGENTINPUTTYPE -eq 0 ]; then
- echo XRD COPY $2 ;
- xrdcp -np $2 data.root
-fi;
-
-
-echo GETTING DATA - STOP
-date
-echo LS DATA
-ls -al
-
-if [ -f data.root ]; then
- ls -al data.root
- echo "File found"
-else
- echo "File not Found"
- exit
-fi
-
-
-
-CISIZE=`ls -l data.root | gawk '{print $5}'`
-if [ $CISIZE -lt 100000 ]; then
- exit
-fi;
-#
-#
-#
-echo BEGIN ACTION
-date
-echo aliroot -b -q $olddir/macros/$4.C
-aliroot -b -q $olddir/macros/$4.C
-echo END ACTION $1
-date
-#
-#
-#
-rm data.root
-rm TPCsignal.root
-ls -al `pwd`
-echo CREATING ZIP FILE
-zip -n root $4 *.root *.log
-
-
-isxrd=`echo $3 | grep root://`
-mkdir $olddir/filelists
-flist=$olddir/filelists/$1.list
-touch $flist
-
-#
-# ALIEN - PROBLEM WITH HOST CERTIFICATE
-#
-if [ ${#alien_HOME} -gt 1 ]; then
- dirname=`echo $alien_HOME$3/$4 | sed s/.root//g `$ASUFIX
- echo alien_mkdir $dirname
- alien_mkdir -p $dirname
- echo ALIEN COPY DATA START
- echo alien_cp -n $4.zip alien:${dirname}/$4_se.zip@$AGENTSE
- for name in `ls *.root`; do
- echo alien_cp -n $name alien:${dirname}/$name@$AGENTSE
- alien_cp -n $name alien:${dirname}/$name@$AGENTSE
- done
- alien_cp -n $4.zip alien:${dirname}/$4_se.zip@$AGENTSE
- echo END OF ALIEN COPY
-fi;
-
-
-
-
-
-cd $olddir
-
+++ /dev/null
-#!/usr/local/bin/bash
-#aguments
-#1 working directory
-
-
-# Example usage:
-# $ALICE_ROOT/TPC/macros/testTPC/agent.sh /afs/cern.ch/user/m/miranov/public/test2008/reckr191xx
-
-# Or submit as many agents you want on batch machines:
-# bsub -q 1nd $ALICE_ROOT/TPC/macros/testTPC/agent.sh /afs/cern.ch/user/m/miranov/public/test2008/reckr191xx
-# myvar=0
-#while [ $myvar -ne 100 ] ; do bsub -q 1nd $ALICE_ROOT/TPC/macros/testTPC/agent.sh /afs/cern.ch/alice/tpctest/test2008/cosmic/37562 ; myvar=$(( $myvar + 1)); echo $myvar; done
-#
-# AGENT to run some action:
-# 3 components to be present
-#
-
-# 0. CREATE ACTION LIST ( see $ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx)
-#
-
-# 1. WORKING DIRECTORY SETUP (in prev. example /afs/cern.ch/user/m/miranov/public/test2008/reckr191xx)
-# 1.a action.list
-# 1.b empty out directory
-# 1.c macros directory -with <action>.C - e.g rec.C FindKrClustersRaw.C
-
-
-# 2. ENVIRONMENT SETUP:
-#
-# 2.a Copy the script to setup aliroot root to the ~/.bagentsetup
-# 2.b Export envirnment variables needed OCDB_PATH, AGENTINPUTTYPE, AGENTSE in the .bagentsetup script
-# 2.c Get a valid token: alien-token-init
-# 2.d Copy certificates to default place - ~/.agentauth/
-# cp /tmp/*$UID* ~/.agentauth/
-
-
-
-#
-#setup environment
-#
-#ALIEN -get token
-echo xxxxxxxxxxxxxxxxxxxxxxxxx
-echo xxxxxxxxxxxxxxxxxxxxxxxxx
-echo xxxxxxxxxxxxxxxxxxxxxxxxx
-echo ENVIRONMENT
-echo ALIEN - $ALIEN
-echo GSHELL_ROOT - $GSHELL_ROOT
-echo HOME - $HOME
-echo ALIROOT =`which aliroot`
-echo ROOT =`which root`
-echo xxxxxxxxxxxxxxxxxxxxxxxxx
-echo xxxxxxxxxxxxxxxxxxxxxxxxx
-echo xxxxxxxxxxxxxxxxxxxxxxxxx
-cp ~/.agentauth/* /tmp/
-source /tmp/gclient_env_$UID
-export PATH=.:${GSHELL_ROOT}/bin:${PATH}.
-
-#aliensh -c ps
-#ALIROOT
-# test if ~/.bagentsetup exists
-if [ ! -f ~/.bagentsetup ]; then
- echo "ERROR: '~/.bagentsetup' init script not found\!"
- exit 1
-fi
-
-source ~/.bagentsetup
-echo xxxxxxxxxxxxxxxxxxxxxxxxx
-echo ENVIRONMENT
-echo ALIROOT =`which aliroot`
-echo ROOT =`which root`
-echo xxxxxxxxxxxxxxxxxxxxxxxxx
-
-
-#chek env
-if [ "x$AGENTINPUTTYPE" == "x" ]; then
- echo "WARNING: environment variable 'AGENTINPUTTYPE' falling back to default (castor)\!\!"
- export AGENTINPUTTYPE=2
-fi
-if [ "x$OCDB_PATH" == "x" ]; then
- echo "WARNING: environment variable 'OCDB_PATH' falling back to default (local:///afs/cern.ch/alice/tpctest/OCDB)\!\!"
- export OCDB_PATH=local:///afs/cern.ch/alice/tpctest/OCDB
-fi
-if [ "x$AGENTSE" == "x" ]; then
- echo "WARNUNG: environment variable 'AGENTSE' falling back to default (ALICE::CERN::SE)\!\!"
- export AGENTSE=ALICE::CERN::SE
-fi
-
-#
-#
-echo OPERATING SYSTEM
-uname -a
-#export workdir
-export jobhome=`pwd`
-
-#cd to workdir and check for 'macros' and 'out' dir
-cd $1
-if [ ! -d macros ]; then
- echo "ERROR: no 'macros' dir in '$1'. Does it exist?"
- exit 1
-fi
-if [ ! -d out ]; then
- echo "ERROR: no 'out' dir in '$1'. Does it exist?"
- exit 1
-fi
-
-# check if we are able to write to this dir, the alien SE, can readback from the alien SE and delete stuff
-touch agend_alien_tst$HOSTNAME.txt || exit 1
-date >> agend_alien_tst$HOSTNAME.txt
-#copy to alien
-alien_cp -dn agend_alien_tst$HOSTNAME.txt alien:${alien_HOME}agend_alien_tst$HOSTNAME.txt@$AGENTSE
-
-#see if it is there
-res=`alien_ls ${alien_HOME}agend_alien_tst$HOSTNAME.txt | grep agend_alien_tst`
-if [ "x$res" == "x" ]; then
- echo "ERROR: could not write on storage element '$AGENTSE'"
- exit 1
-fi
-
-#copy back
-alien_cp -n alien:${alien_HOME}agend_alien_tst$HOSTNAME.txt agend_alien_tst_back.txt
-res=`diff agend_alien_tst$HOSTNAME.txt agend_alien_tst_back.txt`
-if [ "x$res" != "x" ]; then
- echo "ERROR: problems reading from storage element '$AGENTSE'"
- exit 1
-fi
-
-#try to delete from alien
-res=`alien_rm -d ${alien_HOME}agend_alien_tst$HOSTNAME.txt`
-if [ "x$res" != "x" ]; then
- echo "ERROR: cannot delete from storage element '$AGENTSE'"
- exit 1
-fi
-
-#try to delete local files
-res=`rm agend_alien_tst*`
-if [ "x$res" != "x" ]; then
- echo "ERROR: cannot delete local files"
- exit 1
-fi
-
-echo Current Dir
-pwd
-
-aliroot -b -q $ALICE_ROOT/TPC/macros/testTPC/AliTPCjobs.cxx+
-
-#alien_cp -d job.list alien:${alien_HOME}job.list@$AGENTSE
-
+++ /dev/null
-########################################################################
-# The steering script: agent.sh
-#
-# A "small" framework for making fast actions (e.g reconstruction)
-# using the lxbatch facility
-#
-# Author: Marian Ivanov
-#######################################################################
-
-Requirements: 3 components have to be prepeared in order to run the jobs
- 1. WORKING (JOB) DIRECTORY SETUP
- 2. CREATION AND CONTENT OF THE 'job.list'
- 3. ENVIRONMENT SETUP AND JOB SUBMITION
-
-
-1. WORKING (JOB) DIRECTORY SETUP
-(for example /afs/cern.ch/user/m/miranov/public/test2008/cosmic/)
-
- 1.a The 'job.list' (hardwired) has to be created.
- see point 2 on how to create and what it contains
- 1.b Creation of an empty directory called 'out'
- 1.c Creation of a directory called 'macros', which contains your
- specific macros to handle the data
- e.g 'rec.C' 'FindKrClustersRaw.C' etc.
-
-2. CREATION AND CONTENT OF THE 'job.list'
-(for example /afs/cern.ch/user/m/miranov/public/test2008/cosmic/job.list)
-
- Creation of the 'job.list':
- 2.a In a bash shell, setup the root, aliroot and alien version you prefer
- for example by doing
- > source ~miranov/public/.bagentsetup
- Note: The versions have to be avaiable from everywhere (e.g. afs)
- 2.a Get a valid alien token with:
- > alien-token-init
- 2.b source the alien environment and set the server list with
- > source /tmp/gclient_env_$UID
- > export GCLIENT_SERVER_LIST="pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000"
- 2.c create the 'job.list' by following the short example below
- (or example in '$ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx')
-
- Short example (within aliroot do ...):
-
- gSystem->Load("libXrdClient.so");
- gSystem->Load("libNetx.so");
- .L $ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx++
- // 'Run number' and 'search path'
- char *mask = "37562";
- char *datapath = "/alice/data/2008/"
- AlienToolkit toolkit;
- toolkit.MakeCollection(datapath,mask);
- // specify the your 'macro name' and export the list
- char *macroName = "rec";
- toolkit.MakeJobList("job.list","", "", macroName);
-
- 2.d If the first lines of the job.list contains entries like
- '/alice/data/2008/LHC08a_TPC/000028164/collection'
- they need to be deleted manually ...
-
- Content of the 'job.list':
- One line represents one job (one data file) to be handled
- every line consist of 4 columns
- [job name] [input file name] [output directory] [macro name]
-
-3. ENVIRONMENT SETUP AND JOB SUBMITION:
-
- 3.a Copy the script to setup aliroot, root and the user specific
- environment to your afs direcotory (if not already done) ~/.bagentsetup
- (see /afs/cern.ch/user/m/miranov/public/.bagentsetup for an example)
- Note: The needed environment variables like OCDB_PATH, AGENTINPUTTYPE, AGENTSE etc. have to be specified in the .bagentsetup script
- 3.b login to lxplus (64bit), change to a bash shell, create your
- alien token, copy your token to the global accessable location
- (~/.agentauth/ ... hardwired)
- ssh lxplus
- bash
- source ~/.bagentsetup
- alien-token-init
- cp /tmp/*$UID* ~/.agentauth/
- 3.c Submit job localy (important for testing!)
- cd /tmp/$USER (recommanded)
- submit your job with e.g.
- $ALICE_ROOT/TPC/macros/testTPC/agent.sh /afs/cern.ch/user/m/miranov/public/test2008/cosmic/
-
- 3.d Submit as many batch-jobs as you want to the LXBATCH facility with e.g.
- bsub -q 1nd $ALICE_ROOT/TPC/macros/testTPC/agent.sh /afs/cern.ch/user/m/miranov/public/test2008/cosmic/
-
- Check the submitted jobs with the command
- bjobs
- and check the output of your jobs with
- bpeek [jobid] | less
-
- Remarks:
- Every submitted bjob will before he starts to handle a new file look into
- the 'out/' directory in order to check wheter if this file is currently
- running or is already finished. In this case the agent will go on to the
- next "not handled" file.
-
- Note: the 'output directory' in the job.list is hardwired and after the file
- was finished, the output files of your macro will be stored in this
- directory within your alien account. Your output files are therefore
- accessable via the alienshell 'aliensh'.
-
-
---------------------------------------------------------------------------
-REMOTE COPIES (for experts)
-
-Following script generates a bash script stage.sh
-with the xrdcp commands for selected files
-(see AlienToolkit.cxx - AlienToolkit::MakeCollection(dir, mask))
-
-Example:
-gSystem->Load("libXrdClient.so");
-gSystem->Load("libNetx.so");
-.L $ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx+
-AlienToolkit toolkit;
-toolkit.MakeCollection("/alice/cern.ch/user/m/miranov/","08000031843019.1*AliESDs*")
-toolkit.RemoteCopy("/lustre_alpha/alice/TPCdata/cosmics_may2008/",10000000000)
+++ /dev/null
-// #include "TSystem.h"
-// #include "TMath.h"
-// #include "AliRawReader.h"
-// #include "AliRawReaderRoot.h"
-// #include "AliTPCRawStreamFast.h"
-// #include "AliTPCCalibPedestal.h"
-
-void fillPedestal( ){
- AliTPCCalibPedestal *signal = new AliTPCCalibPedestal;
- AliRawReader *rawReader = new AliRawReaderRoot("data.root");
- if ( !rawReader ) return;
-
- rawReader->RewindEvents();
- AliTPCRawStreamFast *strFast = new AliTPCRawStreamFast(rawReader);
-
- while (rawReader->NextEvent()) signal->ProcessEventFast(strFast);
- signal->Analyse();
- signal->DumpToFile("calibPedestal.root");
-
-
- delete strFast;
- delete rawReader;
- delete signal;
-}
+++ /dev/null
-void rec(const char *filename="data.root")
-{
-
- char *ocdbpath = gSystem->Getenv("OCDB_PATH");
- if (ocdbpath==0){
- ocdbpath="alien://folder=/alice/data/2007/LHC07w/OCDB/";
- }
- printf("OCDB PATH = %s\n",ocdbpath);
-
-
- //gSystem->Load("libXrdClient.so");
- //gSystem->Load("libNetx.so");
- AliLog::SetClassDebugLevel("AliTPCRawStream",-5);
- AliLog::SetClassDebugLevel("AliRawReaderDate",-5);
- AliLog::SetClassDebugLevel("AliTPCAltroMapping",-5);
- AliLog::SetModuleDebugLevel("RAW",-5);
- AliLog::SetGlobalLogLevel(3);
-
- //
- // First version of the reconstruction
- // script for the FDR'07
-
- // Set the CDB storage location
- // AliLog::SetModuleDebugLevel("STEER",2);
- AliCDBManager * man = AliCDBManager::Instance();
- //man->SetDefaultStorage("alien://folder=/alice/data/2007/LHC07w/OCDB/");
- man->SetDefaultStorage(ocdbpath);
- // man->SetSpecificStorage("TPC/Calib/Parameters","local:///data/test2007/");
- // man->SetSpecificStorage("TPC/Calib/PadNoise","local:///data/test2007/");
- // man->SetSpecificStorage("ITS/Calib/DDLMapSDD","local://$ALICE_ROOT/OCDB");
- // man->SetSpecificStorage("MUON/Calib/Mapping","local://$ALICE_ROOT/OCDB");
- // man->SetSpecificStorage("MUON/Calib/DDLStore","local://$ALICE_ROOT/OCDB");
-
- // TPC settings
- AliLog::SetClassDebugLevel("AliTPCclustererMI",2);
- AliTPCRecoParam * tpcRecoParam = AliTPCRecoParam::GetCosmicTestParam(kFALSE);
- tpcRecoParam->SetTimeInterval(60,940);
- tpcRecoParam->Dump();
- AliTPCReconstructor::SetRecoParam(tpcRecoParam);
- AliTPCReconstructor::SetStreamLevel(1);
-
- // PHOS settings
- AliPHOSRecoParam* recEmc = new AliPHOSRecoParamEmc();
- recEmc->SetSubtractPedestals(kTRUE);
- recEmc->SetMinE(0.05);
- recEmc->SetClusteringThreshold(0.10);
- AliPHOSReconstructor::SetRecoParamEmc(recEmc);
-
- // T0 settings
- AliLog::SetModuleDebugLevel("T0", 10);
-
- // MUON settings
- AliLog::SetClassDebugLevel("AliMUONRawStreamTracker",3);
-
- // AliReconstruction settings
- AliReconstruction rec;
- rec.SetUniformFieldTracking(kFALSE);
- rec.SetWriteESDfriend(kTRUE);
- rec.SetWriteAlignmentData();
- rec.SetInput(filename);
- //
- //rec.SetRunLocalReconstruction("");
- rec.SetRunReconstruction("TPC");
- rec.SetFillESD("TPC");
- rec.SetRunV0Finder(kFALSE);
- rec.SetRunVertexFinder(kFALSE);
-
- rec.SetRunQA(kTRUE);
-
- // Detector options if any
- rec.SetOption("MUON","SAVEDIGITS");
- rec.SetOption("TPC","OldRCUFormat");
- rec.SetOption("PHOS","OldRCUFormat");
-
- // To be enabled when CTP readout starts
- rec.SetFillTriggerESD(kFALSE);
-
- // all events in one single file
- rec.SetNumberOfEventsPerFile(-1);
-
- // switch off cleanESD
- rec.SetCleanESD(kFALSE);
-
- //AliLog::SetGlobalDebugLevel(2);
- rec.Run();
-
- cout << "-----------------------------------------------------------------" << endl;
- cout << "-----------------------------------------------------------------" << endl;
- cout << "--------- Reconstruction Completed. Start merging QAs -----------" << endl;
- cout << "-----------------------------------------------------------------" << endl;
- cout << "-----------------------------------------------------------------" << endl;
- //
- cout <<" EXITING RECONSTRUNCTION SESSION\n";
- //
- exit();
- AliTPCcalibDB::Instance()->GetParameters()->Dump();
- // AliQADataMakerSteer qas;
- // qas.Merge();
-}