[u/mrichter/AliRoot.git] / ITS / AliITSPreprocessorSPD.cxx

///////////////////////////////////////////////
//  Author: Henrik Tydesjo                   //
//  Preprocessor Class for the SPD           //
//                                           //
///////////////////////////////////////////////

#include "AliITSPreprocessorSPD.h"
#include "AliITSCalibrationSPD.h"
#include "AliITSOnlineCalibrationSPDhandler.h"
#include "AliCDBEntry.h"
#include "AliCDBMetaData.h"
#include "AliShuttleInterface.h"
#include "AliLog.h"
#include <TTimeStamp.h>
#include <TObjString.h>
#include <TSystem.h>
#include <fstream>

/* $Id$ */

ClassImp(AliITSPreprocessorSPD)

//______________________________________________________________________________________________
AliITSPreprocessorSPD::AliITSPreprocessorSPD(AliShuttleInterface* shuttle) :
  AliPreprocessor("SPD", shuttle), fIdList()
{
  // constructor
  fIdList.SetOwner(kTRUE);
}

//______________________________________________________________________________________________
AliITSPreprocessorSPD::~AliITSPreprocessorSPD()
{
  // destructor
}

//______________________________________________________________________________________________
void AliITSPreprocessorSPD::Initialize(Int_t run, UInt_t startTime,
	UInt_t endTime)
{
  // initialize
  AliPreprocessor::Initialize(run, startTime, endTime);

  AliInfo(Form("\n\tRun %d \n\tStartTime %s \n\tEndTime %s", run,
	       TTimeStamp(startTime).AsString(),
	       TTimeStamp(endTime).AsString()));
}

//______________________________________________________________________________________________
UInt_t AliITSPreprocessorSPD::Process(TMap* /*dcsAliasMap*/)
{
  // Do the actual preprocessing


  // *** GET RUN TYPE ***

  TString runType = GetRunType();


  fIdList.Clear();
  UInt_t nrEqForScan  = 0;
  //  UInt_t nrEqForPhysN = 0;
  //  UInt_t nrEqForPhysD = 0;

  // ******************************************************************************************** //
  // *** GET THE FILE IDs FOR DEBUGGING *** //
  if (runType == "DAQ_MIN_TH_SCAN" || 
      runType == "DAQ_MEAN_TH_SCAN" || 
      runType == "DAQ_UNIFORMITY_SCAN" || 
      runType == "DAQ_NOISY_PIX_SCAN" || 
      runType == "DAQ_PIX_DELAY_SCAN" || 
      runType == "DAQ_FO_UNIF_SCAN" || 
      runType == "PHYSICS") {
    TString idListId = "SPD_id_list";
    TList* list = GetFileSources(kDAQ,idListId.Data());
    UInt_t nrIdFiles = 0;
    if (list) {
      TListIter *iter = new TListIter(list);
      while (TObjString* fileNameEntry = (TObjString*) iter->Next()) {
	TString fileName = GetFile(kDAQ, idListId.Data(), fileNameEntry->GetString().Data());
	if (fileName.IsNull()) {
	  Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	  return 1;
	}
	nrIdFiles++;
	ifstream idFile;
	idFile.open(fileName.Data(), ifstream::in);
	if (idFile.fail()) {
	  Log(Form("Could not open file (%s) for reading.",fileName.Data()));
	  return 1;
	}
	else {
	  while(1) {
	    Char_t id[50];
	    idFile >> id;
	    if (idFile.eof()) break;
	    // Add id to the list;
	    fIdList.AddLast(new TObjString(id));
	  }
	}
	idFile.close();
      }
      delete iter;
    }
    if (nrIdFiles==0) {
      Log("Failed to retrieve any id list file.");
      return 1;
    }
  }


  // ******************************************************************************************** //
  // *** REFERENCE DATA *** //

  // Standalone runs:
  if (runType == "DAQ_MIN_TH_SCAN" ||
      runType == "DAQ_MEAN_TH_SCAN" ||
      runType == "DAQ_UNIFORMITY_SCAN" ||
      runType == "DAQ_NOISY_PIX_SCAN" ||
      runType == "DAQ_PIX_DELAY_SCAN" ||
      runType == "DAQ_FO_UNIF_SCAN") {
    // Store the scan container files as reference data (0 or 1 file for each equipment)
    for (UInt_t eq=0; eq<20; eq++) {
      TString id = Form("SPD_ref_scan_%d",eq);
      TList* list = GetFileSources(kDAQ,id.Data()); // (the id should be unique, so always 1 file)
      if (list) {
	TObjString* fileNameEntry = (TObjString*) list->First();
	if (fileNameEntry!=NULL) {
	  nrEqForScan++;
	  TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
	  if (fileName.IsNull()) {
	    Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	    return 1;
	  }
	  if (!RemoveIdFromList(id.Data())) {
	    Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
	  }
	  TString refCAT = Form("SPD_ref_scan_eq_%d",eq);
	  if (!StoreReferenceFile(fileName.Data(),refCAT.Data())) {
	    Log(Form("Failed to store reference file %s.",fileName.Data()));
	    return 1;
	  }
	}
      }
    }
  }

  // Physics runs (online monitoring):
  if (runType == "PHYSICS") {
    // Store the phys "per run" container files as reference data
    if (!StoreRefFromTarForId("SPD_ref_phys")) return 1;
    // Store the phys "dead" container files as reference data
    if (!StoreRefFromTarForId("SPD_ref_phys_dead")) return 1;
  }


// *** OLD CODE NOT TARED
//    for (UInt_t eq=0; eq<20; eq++) {
//      TString id = Form("SPD_ref_phys_%d",eq);
//      TList* list = GetFileSources(kDAQ,id.Data()); // (the id should be unique, so always 1 file)
//      if (list) {
//	TObjString* fileNameEntry = (TObjString*) list->First();
//	if (fileNameEntry!=NULL) {
//	  nrEqForPhysN++;
//	  TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
//	  if (fileName.IsNull()) {
//	    Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
//	    return 1;
//	  }
//	  if (!RemoveIdFromList(id.Data())) {
//	    Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
//	  }
//	  TString refCAT = Form("SPD_ref_phys_eq_%d",eq);
//	  if (!StoreReferenceFile(fileName.Data(),refCAT.Data())) {
//	    Log(Form("Failed to store reference file %s.",fileName.Data()));
//	    return 1;
//	  }
//	}
//      }
//    }
// ***

// *** OLD CODE NOT TARED
//    // Store the phys "dead" container files as reference data (0 or 1 file for each equipment)
//    for (UInt_t eq=0; eq<20; eq++) {
//      TString id = Form("SPD_ref_phys_dead_%d",eq);
//      TList* list = GetFileSources(kDAQ,id.Data()); // (the id should be unique, so always 1 file)
//      if (list) {
//	TObjString* fileNameEntry = (TObjString*) list->First();
//	if (fileNameEntry!=NULL) {
//	  nrEqForPhysD++;
//	  TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
//	  if (fileName.IsNull()) {
//	    Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
//	    return 1;
//	  }
//	  if (!RemoveIdFromList(id.Data())) {
//	    Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
//	  }
//	  TString refCAT = Form("SPD_ref_phys_dead_eq_%d",eq);
//	  if (!StoreReferenceFile(fileName.Data(),refCAT.Data())) {
//	    Log(Form("Failed to store reference file %s.",fileName.Data()));
//	    return 1;
//	  }
//	}
//      }
//    }
//  }
// ***

  // ******************************************************************************************** //


  // *** NOISY AND DEAD DATA *** //

  // Standalone runs:
  if (runType == "DAQ_NOISY_PIX_SCAN") {
    // Retrieve and unpack tared calibration files from FXS
    TString id = "SPD_scan_noisy";
    TList* list = GetFileSources(kDAQ,id.Data());
    if (list) {
      UInt_t index = 0;
      while (list->At(index)!=NULL) {
	TObjString* fileNameEntry = (TObjString*) list->At(index);
	TString fileName = GetFile(kDAQ, id.Data(), fileNameEntry->GetString().Data());
	if (fileName.IsNull()) {
	  Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	  return 1;
	}
	if (!RemoveIdFromList("SPD_scan_noisy")) {
	  Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
	}
	TString command = Form("tar -xf %s",fileName.Data());
	gSystem->Exec(command.Data());
	index++;
      }
    }
    // Create new database entries
    TObjArray* spdEntryNoisy = new TObjArray(240);
    spdEntryNoisy->SetOwner(kTRUE);
    for(UInt_t module=0; module<240; module++){
      AliITSCalibrationSPD* calObj = new AliITSCalibrationSPD();
      spdEntryNoisy->Add(calObj);
    }
    // Add noisy from the copied FXS files
    AliITSOnlineCalibrationSPDhandler* handler = new AliITSOnlineCalibrationSPDhandler();
    TString fileLoc = ".";
    handler->SetFileLocation(fileLoc.Data());
    handler->ReadNoisyFromFiles();
    for (Int_t module=0; module<240; module++) {
      ((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetNrBad( handler->GetNrNoisy(module) );
      ((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetBadList( handler->GetNoisyArray(module) );
    }
    delete handler;
    // Store the new calibration objects in OCDB
    Log("Noisy lists (scan) will be stored...");
    AliCDBMetaData metaData;
    metaData.SetBeamPeriod(0);
    metaData.SetResponsible("Henrik Tydesjo");
    metaData.SetComment("Created by SPD PreProcessor");
    // validity for this run until infinity
    if (!Store("Calib", "SPDNoisy", spdEntryNoisy, &metaData, 0, kTRUE)) {
      Log("Failed to store calibration data.");
      return 1;
    }
    Log("Database updated.");
    delete spdEntryNoisy;
  }

  // Physics runs (online monitoring):
  else if (runType == "PHYSICS") {

    // Noisy pixels:
    // Read noisy from previous calibration
    AliCDBEntry* cdbEntry = GetFromOCDB("Calib", "SPDNoisy");
    TObjArray* spdEntryNoisy;
    if(cdbEntry) {
      spdEntryNoisy = (TObjArray*)cdbEntry->GetObject();
      if(!spdEntryNoisy) return 1;
    }
    else {
      Log("Old calibration not found in database. This is required for further processing.");
      return 1;
    }
    AliITSOnlineCalibrationSPDhandler* handOld = new AliITSOnlineCalibrationSPDhandler();
    handOld->ReadNoisyFromCalibObj(spdEntryNoisy);
    // Retrieve and unpack tared calibration files from FXS
    TString idN = "SPD_phys_noisy";
    TList* listN = GetFileSources(kDAQ,idN.Data());
    if (listN) {
      UInt_t index = 0;
      while (listN->At(index)!=NULL) {
	TObjString* fileNameEntry = (TObjString*) listN->At(index);
	TString fileName = GetFile(kDAQ, idN.Data(), fileNameEntry->GetString().Data());
	if (fileName.IsNull()) {
	  Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	  return 1;
	}
	if (!RemoveIdFromList(idN.Data())) {
	  Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",idN.Data()));
	}
	TString command = Form("tar -xf %s",fileName.Data());
	gSystem->Exec(command.Data());
	index++;
      }
    }
    AliITSOnlineCalibrationSPDhandler* handNew = new AliITSOnlineCalibrationSPDhandler();
    handNew->SetFileLocation(".");
    handNew->ReadNoisyFromFiles();
    // add the new list to the old one
    UInt_t nrNewNoisy = handOld->AddNoisyFrom(handNew);
    // If new noisy pixels were found: Update calibration objects
    if (nrNewNoisy>0) {
      for (Int_t module=0; module<240; module++) {
	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetNrBad( handOld->GetNrNoisy(module) );
	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetBadList( handOld->GetNoisyArray(module) );
      }
      // Store the new calibration objects in OCDB
      Log("Noisy lists (phys) will be stored...");
      AliCDBMetaData metaData;
      metaData.SetBeamPeriod(0);
      metaData.SetResponsible("Henrik Tydesjo");
      metaData.SetComment("Created by SPD PreProcessor");  
      // validity for this run only
      if (!Store("Calib", "SPDNoisy", spdEntryNoisy, &metaData, 0, kFALSE)) {
	Log("Failed to store calibration data.");
	return 1;
      }
      Log("Database updated.");
    }
    delete handNew;

    // Dead pixels:
    // Retrieve and unpack tared calibration files from FXS
    TString idD = "SPD_phys_dead";
    TList* listD = GetFileSources(kDAQ,idD.Data());
    UInt_t nrPhysDeadFiles = 0;
    if (listD) {
      UInt_t index = 0;
      while (listD->At(index)!=NULL) {
	TObjString* fileNameEntry = (TObjString*) listD->At(index);
	TString fileName = GetFile(kDAQ, idD.Data(), fileNameEntry->GetString().Data());
	if (fileName.IsNull()) {
	  Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	  return 1;
	}
	nrPhysDeadFiles++;
	if (!RemoveIdFromList("SPD_phys_dead")) {
	  Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",idD.Data()));
	}
	TString command = Form("tar -xf %s",fileName.Data());
	gSystem->Exec(command.Data());
	index++;
      }
    }
    if (nrPhysDeadFiles==0) {
      Log(Form("Could not find files with id %s. Should be present for each run.",idD.Data()));
      return 1;
    }
    // Create new database entries
    TObjArray* spdEntryDead = new TObjArray(240);
    spdEntryDead->SetOwner(kTRUE);
    for(UInt_t module=0; module<240; module++){
      AliITSCalibrationSPD* calObj = new AliITSCalibrationSPD();
      spdEntryDead->Add(calObj);
    }
    // Add dead from the copied FXS files
    handOld->SetFileLocation(".");
    handOld->ReadDeadFromFiles();
    for (Int_t module=0; module<240; module++) {
      ((AliITSCalibrationSPD*) spdEntryDead->At(module)) -> SetNrBad( handOld->GetNrDead(module) );
      ((AliITSCalibrationSPD*) spdEntryDead->At(module)) -> SetBadList( handOld->GetDeadArray(module) );
    }
    delete handOld;
    // Store the new calibration objects in OCDB
    Log("Dead lists (phys) will be stored...");
    AliCDBMetaData metaData;
    metaData.SetBeamPeriod(0);
    metaData.SetResponsible("Henrik Tydesjo");
    metaData.SetComment("Created by SPD PreProcessor");
    // validity for this run only
    if (!Store("Calib", "SPDDead", spdEntryDead, &metaData, 0, kFALSE)) {
      Log("Failed to store calibration data.");
      return 1;
    }
    Log("Database updated.");
    delete spdEntryDead;

  }


  // check that there are no ids left in the list:
  if (fIdList.First()!=NULL) {
    TString logMessage = "";
    TListIter *iter = new TListIter(&fIdList);
    while (TObjString *st = (TObjString*)iter->Next()) {
      logMessage.Append(st->GetString());
      logMessage.Append(" ");
    }
    delete iter;
    Log(Form("Files with the following ids were never retrieved: %s.",logMessage.Data()));
    return 1;
  }

  fIdList.Clear();

  return 0; // 0 means success

}
//_________________________________________________________________________________________
Bool_t AliITSPreprocessorSPD::RemoveIdFromList(const Char_t *id) {
  // removes id from the list of ids
  Bool_t found = kFALSE;
  TListIter *iter = new TListIter(&fIdList);
  while (TObjString *st = (TObjString*)iter->Next()) {
    if (st->GetString().CompareTo(id)==0) {
      fIdList.Remove(st);
      found = kTRUE;
      break;
    }
  }
  delete iter;
  return found;
}
//_________________________________________________________________________________________
Bool_t AliITSPreprocessorSPD::StoreRefFromTarForId(const Char_t *id) {
  // store reference files from tar file for the id given (this is just to not duplicate code)
  TList* list = GetFileSources(kDAQ,id);
  if (list) {
    UInt_t index = 0;
    while (list->At(index)!=NULL) {
      TObjString* fileNameEntry = (TObjString*) list->At(index);
      TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
      if (fileName.IsNull()) {
	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	return kFALSE;
      }
      if (!RemoveIdFromList(id)) {
	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id));
      }
      // get the file names from the tar file
      //      TString pwd = gSystem->pwd();
      //      TString tempFileName = Form("%s/tempTar.txt",pwd.Data());
      TString tempFileName = "tempTar.txt";
      TString command = Form("tar -tf %s > %s",fileName.Data(),tempFileName.Data());
      gSystem->Exec(command.Data());
      TList fList;
      ifstream tempFile;
      tempFile.open(tempFileName.Data(), ifstream::in);
      if (tempFile.fail()) {
	Log(Form("Could not open file (%s) for reading.",tempFileName.Data()));
	return kFALSE;
      }
      else {
	while(1) {
	  Char_t fileN[100];
	  tempFile >> fileN;
	  if (tempFile.eof()) break;
	  fList.AddLast(new TObjString(fileN));
	}
      }
      // close and remove temp file
      tempFile.close();
      command = Form("rm -f %s",tempFileName.Data());
      gSystem->Exec(command.Data());
      // unpack
      command = Form("tar -xf %s",fileName.Data());
      gSystem->Exec(command.Data());
      // store each file
      UInt_t index2 = 0;
      while (fList.At(index2)!=NULL) {
	TString fileName = ((TObjString*)fList.At(index2))->GetString();
	// get eq id
	TString eqStr = fileName.Data();
	UInt_t len = eqStr.Length();
	eqStr.Replace(0,len-7,"",0);
	eqStr.ReplaceAll("_",1,"",0);
	eqStr.ReplaceAll(".root",5,"",0);
	Int_t eqId = eqStr.Atoi();
	if (eqId>=0 && eqId<20) {
	  TString refCAT = Form("%s_eq_%d",id,eqId);
	  if (!StoreReferenceFile(fileName.Data(),refCAT.Data())) {
	    Log(Form("Failed to store reference file %s.",fileName.Data()));
	    return kFALSE;
	  }
	}
	else {
	  Log(Form("Eq ID %d out of bounds for file %s",eqId,fileName.Data()));
	  fList.Clear();
	  return kFALSE;
	}
	index2++;
      }
      fList.Clear();
      index++;
    }
  }
  return kTRUE;
}
Commit	Line	Data
b15de2d2	1	///////////////////////////////////////////////
	2	// Author: Henrik Tydesjo //
	3	// Preprocessor Class for the SPD //
	4	// //
	5	///////////////////////////////////////////////
3f0e013c	6
3f0e013c	7	#include "AliITSPreprocessorSPD.h"
b15de2d2	8	#include "AliITSCalibrationSPD.h"
b15de2d2	9	#include "AliITSOnlineCalibrationSPDhandler.h"
03fc6773	10	#include "AliCDBEntry.h"
b15de2d2	11	#include "AliCDBMetaData.h"
03fc6773	12	#include "AliShuttleInterface.h"
b15de2d2	13	#include "AliLog.h"
03fc6773	14	#include <TTimeStamp.h>
	15	#include <TObjString.h>
	16	#include <TSystem.h>
4f88491f	17	#include <fstream>
	18
	19	/* $Id$ */
3f0e013c	20
b15de2d2	21	ClassImp(AliITSPreprocessorSPD)
3f0e013c	22
b15de2d2	23	//______________________________________________________________________________________________
b15de2d2	24	AliITSPreprocessorSPD::AliITSPreprocessorSPD(AliShuttleInterface* shuttle) :
4f88491f	25	AliPreprocessor("SPD", shuttle), fIdList()
3f0e013c	26	{
b15de2d2	27	// constructor
4f88491f	28	fIdList.SetOwner(kTRUE);
3f0e013c	29	}
3f0e013c	30
b15de2d2	31	//______________________________________________________________________________________________
b15de2d2	32	AliITSPreprocessorSPD::~AliITSPreprocessorSPD()
3f0e013c	33	{
b15de2d2	34	// destructor
3f0e013c	35	}
3f0e013c	36
b15de2d2	37	//______________________________________________________________________________________________
	38	void AliITSPreprocessorSPD::Initialize(Int_t run, UInt_t startTime,
	39	UInt_t endTime)
3f0e013c	40	{
b15de2d2	41	// initialize
b15de2d2	42	AliPreprocessor::Initialize(run, startTime, endTime);
3f0e013c	43
b15de2d2	44	AliInfo(Form("\n\tRun %d \n\tStartTime %s \n\tEndTime %s", run,
	45	TTimeStamp(startTime).AsString(),
	46	TTimeStamp(endTime).AsString()));
3f0e013c	47	}
fcf95fc7	48
b15de2d2	49	//______________________________________________________________________________________________
0c0426a2	50	UInt_t AliITSPreprocessorSPD::Process(TMap* /dcsAliasMap/)
fcf95fc7	51	{
b15de2d2	52	// Do the actual preprocessing
fcf95fc7	53
03fc6773	54
53ae21ce	55	// * GET RUN TYPE *
ad18504e	56
ad18504e	57	TString runType = GetRunType();
03fc6773	58
4f88491f	59
4f88491f	60	fIdList.Clear();
6727e2db	61	UInt_t nrEqForScan = 0;
4f88491f	62	// UInt_t nrEqForPhysN = 0;
	63	// UInt_t nrEqForPhysD = 0;
	64
	65	// ******************************************************************************************** //
	66	// * GET THE FILE IDs FOR DEBUGGING * //
	67	if (runType == "DAQ_MIN_TH_SCAN" \|\|
	68	runType == "DAQ_MEAN_TH_SCAN" \|\|
	69	runType == "DAQ_UNIFORMITY_SCAN" \|\|
	70	runType == "DAQ_NOISY_PIX_SCAN" \|\|
	71	runType == "DAQ_PIX_DELAY_SCAN" \|\|
	72	runType == "DAQ_FO_UNIF_SCAN" \|\|
	73	runType == "PHYSICS") {
	74	TString idListId = "SPD_id_list";
	75	TList* list = GetFileSources(kDAQ,idListId.Data());
	76	UInt_t nrIdFiles = 0;
	77	if (list) {
	78	TListIter *iter = new TListIter(list);
	79	while (TObjString* fileNameEntry = (TObjString*) iter->Next()) {
	80	TString fileName = GetFile(kDAQ, idListId.Data(), fileNameEntry->GetString().Data());
	81	if (fileName.IsNull()) {
	82	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	83	return 1;
	84	}
	85	nrIdFiles++;
	86	ifstream idFile;
	87	idFile.open(fileName.Data(), ifstream::in);
	88	if (idFile.fail()) {
	89	Log(Form("Could not open file (%s) for reading.",fileName.Data()));
	90	return 1;
	91	}
	92	else {
	93	while(1) {
	94	Char_t id[50];
	95	idFile >> id;
	96	if (idFile.eof()) break;
	97	// Add id to the list;
	98	fIdList.AddLast(new TObjString(id));
	99	}
	100	}
	101	idFile.close();
	102	}
	103	delete iter;
	104	}
	105	if (nrIdFiles==0) {
	106	Log("Failed to retrieve any id list file.");
	107	return 1;
	108	}
	109	}
	110
ad18504e	111
6727e2db	112	// ******************************************************************************************** //
53ae21ce	113	// * REFERENCE DATA * //
	114
	115	// Standalone runs:
	116	if (runType == "DAQ_MIN_TH_SCAN" \|\|
	117	runType == "DAQ_MEAN_TH_SCAN" \|\|
	118	runType == "DAQ_UNIFORMITY_SCAN" \|\|
	119	runType == "DAQ_NOISY_PIX_SCAN" \|\|
	120	runType == "DAQ_PIX_DELAY_SCAN" \|\|
	121	runType == "DAQ_FO_UNIF_SCAN") {
6727e2db	122	// Store the scan container files as reference data (0 or 1 file for each equipment)
53ae21ce	123	for (UInt_t eq=0; eq<20; eq++) {
6727e2db	124	TString id = Form("SPD_ref_scan_%d",eq);
53ae21ce	125	TList* list = GetFileSources(kDAQ,id.Data()); // (the id should be unique, so always 1 file)
	126	if (list) {
	127	TObjString* fileNameEntry = (TObjString*) list->First();
	128	if (fileNameEntry!=NULL) {
6727e2db	129	nrEqForScan++;
53ae21ce	130	TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
4f88491f	131	if (fileName.IsNull()) {
	132	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	133	return 1;
	134	}
	135	if (!RemoveIdFromList(id.Data())) {
	136	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
	137	}
6727e2db	138	TString refCAT = Form("SPD_ref_scan_eq_%d",eq);
53ae21ce	139	if (!StoreReferenceFile(fileName.Data(),refCAT.Data())) {
	140	Log(Form("Failed to store reference file %s.",fileName.Data()));
	141	return 1;
	142	}
0c0426a2	143	}
	144	}
	145	}
	146	}
03fc6773	147
53ae21ce	148	// Physics runs (online monitoring):
53ae21ce	149	if (runType == "PHYSICS") {
4f88491f	150	// Store the phys "per run" container files as reference data
	151	if (!StoreRefFromTarForId("SPD_ref_phys")) return 1;
	152	// Store the phys "dead" container files as reference data
	153	if (!StoreRefFromTarForId("SPD_ref_phys_dead")) return 1;
53ae21ce	154	}
03fc6773	155
4f88491f	156
	157	// *** OLD CODE NOT TARED
	158	// for (UInt_t eq=0; eq<20; eq++) {
	159	// TString id = Form("SPD_ref_phys_%d",eq);
	160	// TList* list = GetFileSources(kDAQ,id.Data()); // (the id should be unique, so always 1 file)
	161	// if (list) {
	162	// TObjString* fileNameEntry = (TObjString*) list->First();
	163	// if (fileNameEntry!=NULL) {
	164	// nrEqForPhysN++;
	165	// TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
	166	// if (fileName.IsNull()) {
	167	// Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	168	// return 1;
	169	// }
	170	// if (!RemoveIdFromList(id.Data())) {
	171	// Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
	172	// }
	173	// TString refCAT = Form("SPD_ref_phys_eq_%d",eq);
	174	// if (!StoreReferenceFile(fileName.Data(),refCAT.Data())) {
	175	// Log(Form("Failed to store reference file %s.",fileName.Data()));
	176	// return 1;
	177	// }
	178	// }
	179	// }
	180	// }
	181	// ***
	182
	183	// *** OLD CODE NOT TARED
	184	// // Store the phys "dead" container files as reference data (0 or 1 file for each equipment)
	185	// for (UInt_t eq=0; eq<20; eq++) {
	186	// TString id = Form("SPD_ref_phys_dead_%d",eq);
	187	// TList* list = GetFileSources(kDAQ,id.Data()); // (the id should be unique, so always 1 file)
	188	// if (list) {
	189	// TObjString* fileNameEntry = (TObjString*) list->First();
	190	// if (fileNameEntry!=NULL) {
	191	// nrEqForPhysD++;
	192	// TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
	193	// if (fileName.IsNull()) {
	194	// Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	195	// return 1;
	196	// }
	197	// if (!RemoveIdFromList(id.Data())) {
	198	// Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
	199	// }
	200	// TString refCAT = Form("SPD_ref_phys_dead_eq_%d",eq);
	201	// if (!StoreReferenceFile(fileName.Data(),refCAT.Data())) {
	202	// Log(Form("Failed to store reference file %s.",fileName.Data()));
	203	// return 1;
	204	// }
	205	// }
	206	// }
	207	// }
	208	// }
	209	// ***
	210
6727e2db	211	// ******************************************************************************************** //
03fc6773	212
03fc6773	213
53ae21ce	214	// * NOISY AND DEAD DATA * //
03fc6773	215
6727e2db	216	// Standalone runs:
	217	if (runType == "DAQ_NOISY_PIX_SCAN") {
	218	// Retrieve and unpack tared calibration files from FXS
4f88491f	219	TString id = "SPD_scan_noisy";
4f88491f	220	TList* list = GetFileSources(kDAQ,id.Data());
6727e2db	221	if (list) {
	222	UInt_t index = 0;
	223	while (list->At(index)!=NULL) {
	224	TObjString* fileNameEntry = (TObjString*) list->At(index);
4f88491f	225	TString fileName = GetFile(kDAQ, id.Data(), fileNameEntry->GetString().Data());
	226	if (fileName.IsNull()) {
	227	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	228	return 1;
	229	}
	230	if (!RemoveIdFromList("SPD_scan_noisy")) {
	231	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
	232	}
6727e2db	233	TString command = Form("tar -xf %s",fileName.Data());
	234	gSystem->Exec(command.Data());
	235	index++;
	236	}
	237	}
	238	// Create new database entries
	239	TObjArray* spdEntryNoisy = new TObjArray(240);
	240	spdEntryNoisy->SetOwner(kTRUE);
	241	for(UInt_t module=0; module<240; module++){
	242	AliITSCalibrationSPD* calObj = new AliITSCalibrationSPD();
	243	spdEntryNoisy->Add(calObj);
	244	}
	245	// Add noisy from the copied FXS files
	246	AliITSOnlineCalibrationSPDhandler* handler = new AliITSOnlineCalibrationSPDhandler();
	247	TString fileLoc = ".";
	248	handler->SetFileLocation(fileLoc.Data());
	249	handler->ReadNoisyFromFiles();
	250	for (Int_t module=0; module<240; module++) {
	251	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetNrBad( handler->GetNrNoisy(module) );
	252	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetBadList( handler->GetNoisyArray(module) );
	253	}
	254	delete handler;
	255	// Store the new calibration objects in OCDB
	256	Log("Noisy lists (scan) will be stored...");
	257	AliCDBMetaData metaData;
	258	metaData.SetBeamPeriod(0);
	259	metaData.SetResponsible("Henrik Tydesjo");
	260	metaData.SetComment("Created by SPD PreProcessor");
	261	// validity for this run until infinity
	262	if (!Store("Calib", "SPDNoisy", spdEntryNoisy, &metaData, 0, kTRUE)) {
	263	Log("Failed to store calibration data.");
	264	return 1;
	265	}
	266	Log("Database updated.");
	267	delete spdEntryNoisy;
	268	}
	269
	270	// Physics runs (online monitoring):
	271	else if (runType == "PHYSICS") {
	272
	273	// Noisy pixels:
	274	// Read noisy from previous calibration
	275	AliCDBEntry* cdbEntry = GetFromOCDB("Calib", "SPDNoisy");
	276	TObjArray* spdEntryNoisy;
53ae21ce	277	if(cdbEntry) {
6727e2db	278	spdEntryNoisy = (TObjArray*)cdbEntry->GetObject();
6727e2db	279	if(!spdEntryNoisy) return 1;
03fc6773	280	}
53ae21ce	281	else {
53ae21ce	282	Log("Old calibration not found in database. This is required for further processing.");
0c0426a2	283	return 1;
0c0426a2	284	}
6727e2db	285	AliITSOnlineCalibrationSPDhandler* handOld = new AliITSOnlineCalibrationSPDhandler();
	286	handOld->ReadNoisyFromCalibObj(spdEntryNoisy);
	287	// Retrieve and unpack tared calibration files from FXS
4f88491f	288	TString idN = "SPD_phys_noisy";
4f88491f	289	TList* listN = GetFileSources(kDAQ,idN.Data());
6727e2db	290	if (listN) {
	291	UInt_t index = 0;
	292	while (listN->At(index)!=NULL) {
	293	TObjString* fileNameEntry = (TObjString*) listN->At(index);
4f88491f	294	TString fileName = GetFile(kDAQ, idN.Data(), fileNameEntry->GetString().Data());
	295	if (fileName.IsNull()) {
	296	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	297	return 1;
	298	}
	299	if (!RemoveIdFromList(idN.Data())) {
	300	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",idN.Data()));
	301	}
6727e2db	302	TString command = Form("tar -xf %s",fileName.Data());
	303	gSystem->Exec(command.Data());
	304	index++;
53ae21ce	305	}
6727e2db	306	}
	307	AliITSOnlineCalibrationSPDhandler* handNew = new AliITSOnlineCalibrationSPDhandler();
	308	handNew->SetFileLocation(".");
	309	handNew->ReadNoisyFromFiles();
	310	// add the new list to the old one
	311	UInt_t nrNewNoisy = handOld->AddNoisyFrom(handNew);
	312	// If new noisy pixels were found: Update calibration objects
	313	if (nrNewNoisy>0) {
53ae21ce	314	for (Int_t module=0; module<240; module++) {
6727e2db	315	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetNrBad( handOld->GetNrNoisy(module) );
6727e2db	316	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetBadList( handOld->GetNoisyArray(module) );
53ae21ce	317	}
6727e2db	318	// Store the new calibration objects in OCDB
	319	Log("Noisy lists (phys) will be stored...");
	320	AliCDBMetaData metaData;
	321	metaData.SetBeamPeriod(0);
	322	metaData.SetResponsible("Henrik Tydesjo");
	323	metaData.SetComment("Created by SPD PreProcessor");
	324	// validity for this run only
	325	if (!Store("Calib", "SPDNoisy", spdEntryNoisy, &metaData, 0, kFALSE)) {
	326	Log("Failed to store calibration data.");
	327	return 1;
53ae21ce	328	}
6727e2db	329	Log("Database updated.");
53ae21ce	330	}
6727e2db	331	delete handNew;
53ae21ce	332
6727e2db	333	// Dead pixels:
6727e2db	334	// Retrieve and unpack tared calibration files from FXS
4f88491f	335	TString idD = "SPD_phys_dead";
	336	TList* listD = GetFileSources(kDAQ,idD.Data());
	337	UInt_t nrPhysDeadFiles = 0;
6727e2db	338	if (listD) {
	339	UInt_t index = 0;
	340	while (listD->At(index)!=NULL) {
	341	TObjString* fileNameEntry = (TObjString*) listD->At(index);
4f88491f	342	TString fileName = GetFile(kDAQ, idD.Data(), fileNameEntry->GetString().Data());
	343	if (fileName.IsNull()) {
	344	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	345	return 1;
	346	}
	347	nrPhysDeadFiles++;
	348	if (!RemoveIdFromList("SPD_phys_dead")) {
	349	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",idD.Data()));
	350	}
6727e2db	351	TString command = Form("tar -xf %s",fileName.Data());
	352	gSystem->Exec(command.Data());
	353	index++;
	354	}
53ae21ce	355	}
4f88491f	356	if (nrPhysDeadFiles==0) {
	357	Log(Form("Could not find files with id %s. Should be present for each run.",idD.Data()));
	358	return 1;
	359	}
6727e2db	360	// Create new database entries
	361	TObjArray* spdEntryDead = new TObjArray(240);
	362	spdEntryDead->SetOwner(kTRUE);
	363	for(UInt_t module=0; module<240; module++){
	364	AliITSCalibrationSPD* calObj = new AliITSCalibrationSPD();
	365	spdEntryDead->Add(calObj);
	366	}
	367	// Add dead from the copied FXS files
	368	handOld->SetFileLocation(".");
	369	handOld->ReadDeadFromFiles();
	370	for (Int_t module=0; module<240; module++) {
	371	((AliITSCalibrationSPD*) spdEntryDead->At(module)) -> SetNrBad( handOld->GetNrDead(module) );
	372	((AliITSCalibrationSPD*) spdEntryDead->At(module)) -> SetBadList( handOld->GetDeadArray(module) );
	373	}
	374	delete handOld;
	375	// Store the new calibration objects in OCDB
	376	Log("Dead lists (phys) will be stored...");
	377	AliCDBMetaData metaData;
	378	metaData.SetBeamPeriod(0);
	379	metaData.SetResponsible("Henrik Tydesjo");
	380	metaData.SetComment("Created by SPD PreProcessor");
	381	// validity for this run only
	382	if (!Store("Calib", "SPDDead", spdEntryDead, &metaData, 0, kFALSE)) {
	383	Log("Failed to store calibration data.");
	384	return 1;
	385	}
	386	Log("Database updated.");
	387	delete spdEntryDead;
53ae21ce	388
03fc6773	389	}
03fc6773	390
03fc6773	391
4f88491f	392	// check that there are no ids left in the list:
	393	if (fIdList.First()!=NULL) {
	394	TString logMessage = "";
	395	TListIter *iter = new TListIter(&fIdList);
	396	while (TObjString st = (TObjString)iter->Next()) {
	397	logMessage.Append(st->GetString());
	398	logMessage.Append(" ");
	399	}
	400	delete iter;
	401	Log(Form("Files with the following ids were never retrieved: %s.",logMessage.Data()));
	402	return 1;
	403	}
	404
	405	fIdList.Clear();
	406
53ae21ce	407	return 0; // 0 means success
03fc6773	408
fcf95fc7	409	}
4f88491f	410	//_________________________________________________________________________________________
	411	Bool_t AliITSPreprocessorSPD::RemoveIdFromList(const Char_t *id) {
	412	// removes id from the list of ids
	413	Bool_t found = kFALSE;
	414	TListIter *iter = new TListIter(&fIdList);
	415	while (TObjString st = (TObjString)iter->Next()) {
	416	if (st->GetString().CompareTo(id)==0) {
	417	fIdList.Remove(st);
	418	found = kTRUE;
	419	break;
	420	}
	421	}
	422	delete iter;
	423	return found;
	424	}
	425	//_________________________________________________________________________________________
	426	Bool_t AliITSPreprocessorSPD::StoreRefFromTarForId(const Char_t *id) {
	427	// store reference files from tar file for the id given (this is just to not duplicate code)
	428	TList* list = GetFileSources(kDAQ,id);
	429	if (list) {
	430	UInt_t index = 0;
	431	while (list->At(index)!=NULL) {
	432	TObjString* fileNameEntry = (TObjString*) list->At(index);
	433	TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
	434	if (fileName.IsNull()) {
	435	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	436	return kFALSE;
	437	}
	438	if (!RemoveIdFromList(id)) {
	439	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id));
	440	}
	441	// get the file names from the tar file
	442	// TString pwd = gSystem->pwd();
	443	// TString tempFileName = Form("%s/tempTar.txt",pwd.Data());
	444	TString tempFileName = "tempTar.txt";
	445	TString command = Form("tar -tf %s > %s",fileName.Data(),tempFileName.Data());
	446	gSystem->Exec(command.Data());
	447	TList fList;
	448	ifstream tempFile;
	449	tempFile.open(tempFileName.Data(), ifstream::in);
	450	if (tempFile.fail()) {
	451	Log(Form("Could not open file (%s) for reading.",tempFileName.Data()));
	452	return kFALSE;
	453	}
	454	else {
	455	while(1) {
	456	Char_t fileN[100];
	457	tempFile >> fileN;
	458	if (tempFile.eof()) break;
	459	fList.AddLast(new TObjString(fileN));
	460	}
	461	}
	462	// close and remove temp file
	463	tempFile.close();
	464	command = Form("rm -f %s",tempFileName.Data());
	465	gSystem->Exec(command.Data());
	466	// unpack
	467	command = Form("tar -xf %s",fileName.Data());
	468	gSystem->Exec(command.Data());
	469	// store each file
	470	UInt_t index2 = 0;
	471	while (fList.At(index2)!=NULL) {
	472	TString fileName = ((TObjString*)fList.At(index2))->GetString();
	473	// get eq id
474	TString eqStr = fileName.Data();
475	UInt_t len = eqStr.Length();
476	eqStr.Replace(0,len-7,"",0);
477	eqStr.ReplaceAll("_",1,"",0);
478	eqStr.ReplaceAll(".root",5,"",0);
479	Int_t eqId = eqStr.Atoi();
480	if (eqId>=0 && eqId<20) {
481	TString refCAT = Form("%s_eq_%d",id,eqId);
482	if (!StoreReferenceFile(fileName.Data(),refCAT.Data())) {
483	Log(Form("Failed to store reference file %s.",fileName.Data()));
484	return kFALSE;
485	}
486	}
487	else {
488	Log(Form("Eq ID %d out of bounds for file %s",eqId,fileName.Data()));
489	fList.Clear();
490	return kFALSE;
491	}
492	index2++;
493	}
494	fList.Clear();
495	index++;
496	}
497	}
498	return kTRUE;
499	}