[u/mrichter/AliRoot.git] / ITS / AliITSPreprocessorSPD.cxx

///////////////////////////////////////////////
//  Author: Henrik Tydesjo                   //
//  Preprocessor Class for the SPD           //
//                                           //
///////////////////////////////////////////////

#include "AliITSPreprocessorSPD.h"
#include "AliITSCalibrationSPD.h"
#include "AliITSOnlineCalibrationSPDhandler.h"
#include "AliCDBEntry.h"
#include "AliCDBMetaData.h"
#include "AliShuttleInterface.h"
#include "AliLog.h"
#include <TTimeStamp.h>
#include <TObjString.h>
#include <TSystem.h>
#include <fstream>

/* $Id$ */

using std::ifstream;
ClassImp(AliITSPreprocessorSPD)

//______________________________________________________________________________________________
AliITSPreprocessorSPD::AliITSPreprocessorSPD(AliShuttleInterface* shuttle) :
  AliPreprocessor("SPD", shuttle), fIdList()
{
  // constructor
  AddRunType("DAQ_MIN_TH_SCAN");
  AddRunType("DAQ_MEAN_TH_SCAN");
  AddRunType("DAQ_GEN_DAC_SCAN");
  AddRunType("DAQ_UNIFORMITY_SCAN");
  AddRunType("DAQ_NOISY_PIX_SCAN");
  AddRunType("DAQ_PIX_DELAY_SCAN");
  AddRunType("DAQ_FO_UNIF_SCAN");
  AddRunType("PHYSICS");

  fIdList.SetOwner(kTRUE);
}

//______________________________________________________________________________________________
AliITSPreprocessorSPD::~AliITSPreprocessorSPD()
{
  // destructor
}

//______________________________________________________________________________________________
void AliITSPreprocessorSPD::Initialize(Int_t run, UInt_t startTime,
	UInt_t endTime)
{
  // initialize
  AliPreprocessor::Initialize(run, startTime, endTime);

  AliInfo(Form("\n\tRun %d \n\tStartTime %s \n\tEndTime %s", run,
	       TTimeStamp(startTime).AsString(),
	       TTimeStamp(endTime).AsString()));
}

//______________________________________________________________________________________________
UInt_t AliITSPreprocessorSPD::Process(TMap* /*dcsAliasMap*/)
{
  // Do the actual preprocessing


  // *** GET RUN TYPE ***

  TString runType = GetRunType();


  fIdList.Clear();


  // ******************************************************************************************** //
  // *** GET THE FILE IDs FOR DEBUGGING *** //
  if (runType == "DAQ_MIN_TH_SCAN" || 
      runType == "DAQ_MEAN_TH_SCAN" || 
      runType == "DAQ_GEN_DAC_SCAN" || 
      runType == "DAQ_UNIFORMITY_SCAN" || 
      runType == "DAQ_NOISY_PIX_SCAN" || 
      runType == "DAQ_PIX_DELAY_SCAN" || 
      runType == "DAQ_FO_UNIF_SCAN" || 
      runType == "PHYSICS") {
    TString idListId = "SPD_id_list";
    TList* list = GetFileSources(kDAQ,idListId.Data());
    UInt_t nrIdFiles = 0;
    if (list) {
      TListIter *iter = new TListIter(list);
      while (TObjString* fileNameEntry = (TObjString*) iter->Next()) {
	TString fileName = GetFile(kDAQ, idListId.Data(), fileNameEntry->GetString().Data());
	if (fileName.IsNull()) {
	  Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	  return 1;
	}
	nrIdFiles++;
	ifstream idFile;
	idFile.open(fileName.Data(), ifstream::in);
	if (idFile.fail()) {
	  Log(Form("Could not open file (%s) for reading.",fileName.Data()));
	  return 1;
	}
	else {
	  while(1) {
	    Char_t id[50];
	    idFile >> id;
	    if (idFile.eof()) break;
	    // Add id to the list;
	    fIdList.AddLast(new TObjString(id));
	  }
	}
	idFile.close();
      }
      delete iter;
    }
    if (nrIdFiles==0) {
      Log("Failed to retrieve any id list file.");
      return 1;
    }
  }


  // ******************************************************************************************** //
  // *** REFERENCE DATA *** //

  // SCAN runs:
  if (runType == "DAQ_MIN_TH_SCAN" ||
      runType == "DAQ_MEAN_TH_SCAN" ||
      runType == "DAQ_GEN_DAC_SCAN" || 
      runType == "DAQ_UNIFORMITY_SCAN" ||
      runType == "DAQ_NOISY_PIX_SCAN" ||
      runType == "DAQ_PIX_DELAY_SCAN") {
    // Store scan container files for all equipments used - as reference data
    // ids from FXS follow ("SPD_ref_scan_%d",eq), while Alien follow ("SPD_ref_scan_eq_%d",eq)
    // the first part of the id is passed as argument here:
    if (!StoreRefForIdStartingWith("SPD_ref_scan")) return 1;
  }

  // FO runs:
  else if (runType == "DAQ_FO_UNIF_SCAN") {
    // Store fo container files for all equipments used - as reference data
    // ids from FXS follow ("SPD_ref_fo_%d",eq), while Alien follow ("SPD_ref_fo_eq_%d",eq)
    // the first part of the id is passed as argument here:
    if (!StoreRefForIdStartingWith("SPD_ref_fo")) return 1;
  }

  // Physics runs (online monitoring):
  else if (runType == "PHYSICS") {
    // Store the phys "per run" container files - as reference data
    if (!StoreRefFromTarForId("SPD_ref_phys")) return 1;
    // Store the phys "dead" container files - as reference data
    if (!StoreRefFromTarForId("SPD_ref_phys_dead")) return 1;
  }


  // ******************************************************************************************** //
  // *** NOISY AND DEAD DATA *** //

  // Standalone runs:
  if (runType == "DAQ_NOISY_PIX_SCAN") {
    // Retrieve and unpack tared calibration files from FXS
    TString id = "SPD_scan_noisy";
    TList* list = GetFileSources(kDAQ,id.Data());
    if (list) {
      UInt_t index = 0;
      while (list->At(index)!=NULL) {
	TObjString* fileNameEntry = (TObjString*) list->At(index);
	TString fileName = GetFile(kDAQ, id.Data(), fileNameEntry->GetString().Data());
	if (fileName.IsNull()) {
	  Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	  return 1;
	}
	if (!RemoveIdFromList("SPD_scan_noisy")) {
	  Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
	}
	TString command = Form("tar -xf %s",fileName.Data());
	gSystem->Exec(command.Data());
	index++;
      }
    }
    // Create new database entries
    TObjArray* spdEntryNoisy = new TObjArray(240);
    spdEntryNoisy->SetOwner(kTRUE);
    for(UInt_t module=0; module<240; module++){
      AliITSCalibrationSPD* calObj = new AliITSCalibrationSPD();
      spdEntryNoisy->Add(calObj);
    }
    // Add noisy from the copied FXS files
    AliITSOnlineCalibrationSPDhandler* handler = new AliITSOnlineCalibrationSPDhandler();
    TString fileLoc = ".";
    handler->SetFileLocation(fileLoc.Data());
    handler->ReadNoisyFromFiles();
    for (Int_t module=0; module<240; module++) {
      ((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetNrBadSingle( handler->GetNrNoisySingle(module) );
      ((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetBadList( handler->GetNoisyArray(module) );
    }
    delete handler;
    // Store the new calibration objects in OCDB
    Log("Noisy lists (scan) will be stored...");
    AliCDBMetaData metaData;
    metaData.SetBeamPeriod(0);
    metaData.SetResponsible("Henrik Tydesjo");
    metaData.SetComment("Created by SPD PreProcessor");
    // validity for this run until infinity
    if (!Store("Calib", "SPDNoisy", spdEntryNoisy, &metaData, 0, kTRUE)) {
      Log("Failed to store calibration data.");
      return 1;
    }
    Log("Database updated.");
    delete spdEntryNoisy;
  }

  // Physics runs (online monitoring):
  else if (runType == "PHYSICS") {

    // Noisy pixels:
    // Read noisy from previous calibration
    AliCDBEntry* cdbEntry = GetFromOCDB("Calib", "SPDNoisy");
    TObjArray* spdEntryNoisy;
    if(cdbEntry) {
      spdEntryNoisy = (TObjArray*)cdbEntry->GetObject();
      if(!spdEntryNoisy) return 1;
    }
    else {
      Log("Old calibration not found in database. This is required for further processing.");
      return 1;
    }
    AliITSOnlineCalibrationSPDhandler* handOld = new AliITSOnlineCalibrationSPDhandler();
    handOld->ReadNoisyFromCalibObj(spdEntryNoisy);
    // Retrieve and unpack tared calibration files from FXS
    TString idN = "SPD_phys_noisy";
    TList* listN = GetFileSources(kDAQ,idN.Data());
    if (listN) {
      UInt_t index = 0;
      while (listN->At(index)!=NULL) {
	TObjString* fileNameEntry = (TObjString*) listN->At(index);
	TString fileName = GetFile(kDAQ, idN.Data(), fileNameEntry->GetString().Data());
	if (fileName.IsNull()) {
	  Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	  return 1;
	}
	if (!RemoveIdFromList(idN.Data())) {
	  Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",idN.Data()));
	}
	TString command = Form("tar -xf %s",fileName.Data());
	gSystem->Exec(command.Data());
	index++;
      }
    }
    AliITSOnlineCalibrationSPDhandler* handNew = new AliITSOnlineCalibrationSPDhandler();
    handNew->SetFileLocation(".");
    handNew->ReadNoisyFromFiles();
    // add the new list to the old one
    UInt_t nrNewNoisy = handOld->AddNoisyFrom(handNew);
    // If new noisy pixels were found: Update calibration objects
    if (nrNewNoisy>0) {
      for (Int_t module=0; module<240; module++) {
	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetNrBadSingle( handOld->GetNrNoisySingle(module) );
	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetBadList( handOld->GetNoisyArray(module) );
      }
      // Store the new calibration objects in OCDB
      Log("Noisy lists (phys) will be stored...");
      AliCDBMetaData metaData;
      metaData.SetBeamPeriod(0);
      metaData.SetResponsible("Henrik Tydesjo");
      metaData.SetComment("Created by SPD PreProcessor");  
      // validity for this run only
      if (!Store("Calib", "SPDNoisy", spdEntryNoisy, &metaData, 0, kFALSE)) {
	Log("Failed to store calibration data.");
	return 1;
      }
      Log("Database updated.");
    }
    delete handNew;

    // Dead pixels:
    // Retrieve and unpack tared calibration files from FXS
    TString idD = "SPD_phys_dead";
    TList* listD = GetFileSources(kDAQ,idD.Data());
    UInt_t nrPhysDeadFiles = 0;
    if (listD) {
      UInt_t index = 0;
      while (listD->At(index)!=NULL) {
	TObjString* fileNameEntry = (TObjString*) listD->At(index);
	TString fileName = GetFile(kDAQ, idD.Data(), fileNameEntry->GetString().Data());
	if (fileName.IsNull()) {
	  Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	  return 1;
	}
	nrPhysDeadFiles++;
	if (!RemoveIdFromList("SPD_phys_dead")) {
	  Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",idD.Data()));
	}
	TString command = Form("tar -xf %s",fileName.Data());
	gSystem->Exec(command.Data());
	index++;
      }
    }
    if (nrPhysDeadFiles==0) {
      Log(Form("Could not find files with id %s. Should be present for each run.",idD.Data()));
      return 1;
    }
    // Create new database entries
    TObjArray* spdEntryDead = new TObjArray(240);
    spdEntryDead->SetOwner(kTRUE);
    for(UInt_t module=0; module<240; module++){
      AliITSCalibrationSPD* calObj = new AliITSCalibrationSPD();
      spdEntryDead->Add(calObj);
    }
    // Add dead from the copied FXS files
    handOld->SetFileLocation(".");
    handOld->ReadSilentFromFiles();
    for (UInt_t module=0; module<240; module++) {
      AliITSCalibrationSPD* calibSPD = (AliITSCalibrationSPD*) spdEntryDead->At(module);
      calibSPD->SetNrBadSingle( handOld->GetNrDeadSingle(module) );
      calibSPD->SetBadList( handOld->GetDeadArray(module) );
      for (UInt_t chipIndex=0; chipIndex<5; chipIndex++) {
	UInt_t eq,hs,chip,col,row;
	AliITSRawStreamSPD::OfflineToOnline(module, chipIndex*32, 0, eq, hs, chip, col, row);
	if (handOld->IsSilentChip(eq,hs,chip)) {
	  calibSPD->SetChipBad(chipIndex);
	}
	else {
	  calibSPD->UnSetChipBad(chipIndex);
	}
      }
    }
    delete handOld;
    // Store the new calibration objects in OCDB
    Log("Dead lists (phys) will be stored...");
    AliCDBMetaData metaData;
    metaData.SetBeamPeriod(0);
    metaData.SetResponsible("Henrik Tydesjo");
    metaData.SetComment("Created by SPD PreProcessor");
    // validity for this run only
    if (!Store("Calib", "SPDDead", spdEntryDead, &metaData, 0, kFALSE)) {
      Log("Failed to store calibration data.");
      return 1;
    }
    Log("Database updated.");
    delete spdEntryDead;

  }


  // ******************************************************************************************** //
  // check that there are no ids left in the list:
  if (fIdList.First()!=NULL) {
    TString logMessage = "";
    TListIter *iter = new TListIter(&fIdList);
    while (TObjString *st = (TObjString*)iter->Next()) {
      logMessage.Append(st->GetString());
      logMessage.Append(" ");
    }
    delete iter;
    Log(Form("Files with the following ids were never retrieved: %s.",logMessage.Data()));
    return 1;
  }
  fIdList.Clear();


  return 0; // 0 means success

}
//_________________________________________________________________________________________
Bool_t AliITSPreprocessorSPD::RemoveIdFromList(const Char_t *id) {
  // removes id from the list of ids
  Bool_t found = kFALSE;
  TListIter *iter = new TListIter(&fIdList);
  while (TObjString *st = (TObjString*)iter->Next()) {
    if (st->GetString().CompareTo(id)==0) {
      fIdList.Remove(st);
      found = kTRUE;
      break;
    }
  }
  delete iter;
  return found;
}
//_________________________________________________________________________________________
Bool_t AliITSPreprocessorSPD::StoreRefForIdStartingWith(const Char_t *idStart) {
  // Store the standalone container files as reference data (0 or 1 file for each equipment)
  // idStart is the first part of the id string (which also should contain the actual eq)
  for (UInt_t eq=0; eq<20; eq++) {
    TString id = Form("%s_%d",idStart,eq);
    TList* list = GetFileSources(kDAQ,id.Data()); // (the id should be unique, so always 1 file)
    if (list) {
      TObjString* fileNameEntry = (TObjString*) list->First();
      if (fileNameEntry!=NULL) {
	TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
	if (fileName.IsNull()) {
	    Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	    return kFALSE;
	}
	if (!RemoveIdFromList(id.Data())) {
	  Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
	}
	TString refCAT = Form("%s_eq_%d",idStart,eq);
	if (!StoreReferenceFile(fileName.Data(),refCAT.Data())) {
	  Log(Form("Failed to store reference file %s.",fileName.Data()));
	  return kFALSE;
	}
      }
    }
  }
  return kTRUE;
}
//_________________________________________________________________________________________
Bool_t AliITSPreprocessorSPD::StoreRefFromTarForId(const Char_t *id) {
  // store reference files from tar file for the id given (this is just to not duplicate code)
  TList* list = GetFileSources(kDAQ,id);
  if (list) {
    UInt_t index = 0;
    while (list->At(index)!=NULL) {
      TObjString* fileNameEntry = (TObjString*) list->At(index);
      TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
      if (fileName.IsNull()) {
	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	return kFALSE;
      }
      if (!RemoveIdFromList(id)) {
	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id));
      }
      // get the file names from the tar file
      //      TString pwd = gSystem->pwd();
      //      TString tempFileName = Form("%s/tempTar.txt",pwd.Data());
      TString tempFileName = "tempTar.txt";
      TString command = Form("tar -tf %s > %s",fileName.Data(),tempFileName.Data());
      gSystem->Exec(command.Data());
      TList fList;
      ifstream tempFile;
      tempFile.open(tempFileName.Data(), ifstream::in);
      if (tempFile.fail()) {
	Log(Form("Could not open file (%s) for reading.",tempFileName.Data()));
	return kFALSE;
      }
      else {
	while(1) {
	  Char_t fileN[100];
	  tempFile >> fileN;
	  if (tempFile.eof()) break;
	  fList.AddLast(new TObjString(fileN));
	}
      }
      // close and remove temp file
      tempFile.close();
      command = Form("rm -f %s",tempFileName.Data());
      gSystem->Exec(command.Data());
      // unpack
      command = Form("tar -xf %s",fileName.Data());
      gSystem->Exec(command.Data());
      // store each file
      UInt_t index2 = 0;
      while (fList.At(index2)!=NULL) {
	TString eqFileName = ((TObjString*)fList.At(index2))->GetString();
	// get eq id
	TString eqStr = eqFileName.Data();
	UInt_t len = eqStr.Length();
	eqStr.Replace(0,len-7,"",0);
	eqStr.ReplaceAll("_",1,"",0);
	eqStr.ReplaceAll(".root",5,"",0);
	Int_t eqId = eqStr.Atoi();
	if (eqId>=0 && eqId<20) {
	  TString refCAT = Form("%s_eq_%d",id,eqId);
	  if (!StoreReferenceFile(eqFileName.Data(),refCAT.Data())) {
	    Log(Form("Failed to store reference file %s.",eqFileName.Data()));
	    return kFALSE;
	  }
	}
	else {
	  Log(Form("Eq ID %d out of bounds for file %s",eqId,eqFileName.Data()));
	  fList.Clear();
	  return kFALSE;
	}
	index2++;
      }
      fList.Clear();
      index++;
    }
  }
  return kTRUE;
}
Commit	Line	Data
b15de2d2	1	///////////////////////////////////////////////
	2	// Author: Henrik Tydesjo //
	3	// Preprocessor Class for the SPD //
	4	// //
	5	///////////////////////////////////////////////
3f0e013c	6
3f0e013c	7	#include "AliITSPreprocessorSPD.h"
b15de2d2	8	#include "AliITSCalibrationSPD.h"
b15de2d2	9	#include "AliITSOnlineCalibrationSPDhandler.h"
03fc6773	10	#include "AliCDBEntry.h"
b15de2d2	11	#include "AliCDBMetaData.h"
03fc6773	12	#include "AliShuttleInterface.h"
b15de2d2	13	#include "AliLog.h"
03fc6773	14	#include <TTimeStamp.h>
	15	#include <TObjString.h>
	16	#include <TSystem.h>
4f88491f	17	#include <fstream>
	18
	19	/* $Id$ */
3f0e013c	20
fe7d86eb	21	using std::ifstream;
b15de2d2	22	ClassImp(AliITSPreprocessorSPD)
3f0e013c	23
b15de2d2	24	//______________________________________________________________________________________________
b15de2d2	25	AliITSPreprocessorSPD::AliITSPreprocessorSPD(AliShuttleInterface* shuttle) :
4f88491f	26	AliPreprocessor("SPD", shuttle), fIdList()
3f0e013c	27	{
b15de2d2	28	// constructor
b2435c94	29	AddRunType("DAQ_MIN_TH_SCAN");
b2435c94	30	AddRunType("DAQ_MEAN_TH_SCAN");
ad07f1ab	31	AddRunType("DAQ_GEN_DAC_SCAN");
b2435c94	32	AddRunType("DAQ_UNIFORMITY_SCAN");
	33	AddRunType("DAQ_NOISY_PIX_SCAN");
	34	AddRunType("DAQ_PIX_DELAY_SCAN");
	35	AddRunType("DAQ_FO_UNIF_SCAN");
	36	AddRunType("PHYSICS");
	37
4f88491f	38	fIdList.SetOwner(kTRUE);
3f0e013c	39	}
3f0e013c	40
b15de2d2	41	//______________________________________________________________________________________________
b15de2d2	42	AliITSPreprocessorSPD::~AliITSPreprocessorSPD()
3f0e013c	43	{
b15de2d2	44	// destructor
3f0e013c	45	}
3f0e013c	46
b15de2d2	47	//______________________________________________________________________________________________
	48	void AliITSPreprocessorSPD::Initialize(Int_t run, UInt_t startTime,
	49	UInt_t endTime)
3f0e013c	50	{
b15de2d2	51	// initialize
b15de2d2	52	AliPreprocessor::Initialize(run, startTime, endTime);
3f0e013c	53
b15de2d2	54	AliInfo(Form("\n\tRun %d \n\tStartTime %s \n\tEndTime %s", run,
	55	TTimeStamp(startTime).AsString(),
	56	TTimeStamp(endTime).AsString()));
3f0e013c	57	}
fcf95fc7	58
b15de2d2	59	//______________________________________________________________________________________________
0c0426a2	60	UInt_t AliITSPreprocessorSPD::Process(TMap* /dcsAliasMap/)
fcf95fc7	61	{
b15de2d2	62	// Do the actual preprocessing
fcf95fc7	63
03fc6773	64
53ae21ce	65	// * GET RUN TYPE *
ad18504e	66
ad18504e	67	TString runType = GetRunType();
03fc6773	68
4f88491f	69
4f88491f	70	fIdList.Clear();
af866a70	71
4f88491f	72
	73	// ******************************************************************************************** //
	74	// * GET THE FILE IDs FOR DEBUGGING * //
	75	if (runType == "DAQ_MIN_TH_SCAN" \|\|
	76	runType == "DAQ_MEAN_TH_SCAN" \|\|
ad07f1ab	77	runType == "DAQ_GEN_DAC_SCAN" \|\|
4f88491f	78	runType == "DAQ_UNIFORMITY_SCAN" \|\|
	79	runType == "DAQ_NOISY_PIX_SCAN" \|\|
	80	runType == "DAQ_PIX_DELAY_SCAN" \|\|
	81	runType == "DAQ_FO_UNIF_SCAN" \|\|
	82	runType == "PHYSICS") {
	83	TString idListId = "SPD_id_list";
	84	TList* list = GetFileSources(kDAQ,idListId.Data());
	85	UInt_t nrIdFiles = 0;
	86	if (list) {
	87	TListIter *iter = new TListIter(list);
	88	while (TObjString* fileNameEntry = (TObjString*) iter->Next()) {
	89	TString fileName = GetFile(kDAQ, idListId.Data(), fileNameEntry->GetString().Data());
	90	if (fileName.IsNull()) {
	91	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	92	return 1;
	93	}
	94	nrIdFiles++;
	95	ifstream idFile;
	96	idFile.open(fileName.Data(), ifstream::in);
	97	if (idFile.fail()) {
	98	Log(Form("Could not open file (%s) for reading.",fileName.Data()));
	99	return 1;
	100	}
	101	else {
	102	while(1) {
	103	Char_t id[50];
	104	idFile >> id;
	105	if (idFile.eof()) break;
	106	// Add id to the list;
	107	fIdList.AddLast(new TObjString(id));
	108	}
	109	}
	110	idFile.close();
	111	}
	112	delete iter;
	113	}
	114	if (nrIdFiles==0) {
	115	Log("Failed to retrieve any id list file.");
	116	return 1;
	117	}
	118	}
	119
ad18504e	120
6727e2db	121	// ******************************************************************************************** //
53ae21ce	122	// * REFERENCE DATA * //
53ae21ce	123
af866a70	124	// SCAN runs:
53ae21ce	125	if (runType == "DAQ_MIN_TH_SCAN" \|\|
53ae21ce	126	runType == "DAQ_MEAN_TH_SCAN" \|\|
ad07f1ab	127	runType == "DAQ_GEN_DAC_SCAN" \|\|
53ae21ce	128	runType == "DAQ_UNIFORMITY_SCAN" \|\|
53ae21ce	129	runType == "DAQ_NOISY_PIX_SCAN" \|\|
af866a70	130	runType == "DAQ_PIX_DELAY_SCAN") {
	131	// Store scan container files for all equipments used - as reference data
	132	// ids from FXS follow ("SPD_ref_scan_%d",eq), while Alien follow ("SPD_ref_scan_eq_%d",eq)
	133	// the first part of the id is passed as argument here:
	134	if (!StoreRefForIdStartingWith("SPD_ref_scan")) return 1;
	135	}
	136
	137	// FO runs:
	138	else if (runType == "DAQ_FO_UNIF_SCAN") {
	139	// Store fo container files for all equipments used - as reference data
	140	// ids from FXS follow ("SPD_ref_fo_%d",eq), while Alien follow ("SPD_ref_fo_eq_%d",eq)
	141	// the first part of the id is passed as argument here:
	142	if (!StoreRefForIdStartingWith("SPD_ref_fo")) return 1;
0c0426a2	143	}
03fc6773	144
53ae21ce	145	// Physics runs (online monitoring):
af866a70	146	else if (runType == "PHYSICS") {
af866a70	147	// Store the phys "per run" container files - as reference data
4f88491f	148	if (!StoreRefFromTarForId("SPD_ref_phys")) return 1;
af866a70	149	// Store the phys "dead" container files - as reference data
4f88491f	150	if (!StoreRefFromTarForId("SPD_ref_phys_dead")) return 1;
53ae21ce	151	}
03fc6773	152
4f88491f	153
4f88491f	154
6727e2db	155	// ******************************************************************************************** //
53ae21ce	156	// * NOISY AND DEAD DATA * //
03fc6773	157
6727e2db	158	// Standalone runs:
	159	if (runType == "DAQ_NOISY_PIX_SCAN") {
	160	// Retrieve and unpack tared calibration files from FXS
4f88491f	161	TString id = "SPD_scan_noisy";
4f88491f	162	TList* list = GetFileSources(kDAQ,id.Data());
6727e2db	163	if (list) {
	164	UInt_t index = 0;
	165	while (list->At(index)!=NULL) {
	166	TObjString* fileNameEntry = (TObjString*) list->At(index);
4f88491f	167	TString fileName = GetFile(kDAQ, id.Data(), fileNameEntry->GetString().Data());
	168	if (fileName.IsNull()) {
	169	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	170	return 1;
	171	}
	172	if (!RemoveIdFromList("SPD_scan_noisy")) {
	173	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
	174	}
6727e2db	175	TString command = Form("tar -xf %s",fileName.Data());
	176	gSystem->Exec(command.Data());
	177	index++;
	178	}
	179	}
	180	// Create new database entries
	181	TObjArray* spdEntryNoisy = new TObjArray(240);
	182	spdEntryNoisy->SetOwner(kTRUE);
	183	for(UInt_t module=0; module<240; module++){
	184	AliITSCalibrationSPD* calObj = new AliITSCalibrationSPD();
	185	spdEntryNoisy->Add(calObj);
	186	}
	187	// Add noisy from the copied FXS files
	188	AliITSOnlineCalibrationSPDhandler* handler = new AliITSOnlineCalibrationSPDhandler();
	189	TString fileLoc = ".";
	190	handler->SetFileLocation(fileLoc.Data());
	191	handler->ReadNoisyFromFiles();
	192	for (Int_t module=0; module<240; module++) {
4821f07d	193	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetNrBadSingle( handler->GetNrNoisySingle(module) );
6727e2db	194	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetBadList( handler->GetNoisyArray(module) );
	195	}
	196	delete handler;
	197	// Store the new calibration objects in OCDB
	198	Log("Noisy lists (scan) will be stored...");
	199	AliCDBMetaData metaData;
	200	metaData.SetBeamPeriod(0);
	201	metaData.SetResponsible("Henrik Tydesjo");
	202	metaData.SetComment("Created by SPD PreProcessor");
	203	// validity for this run until infinity
	204	if (!Store("Calib", "SPDNoisy", spdEntryNoisy, &metaData, 0, kTRUE)) {
	205	Log("Failed to store calibration data.");
	206	return 1;
	207	}
	208	Log("Database updated.");
	209	delete spdEntryNoisy;
	210	}
	211
	212	// Physics runs (online monitoring):
	213	else if (runType == "PHYSICS") {
	214
	215	// Noisy pixels:
	216	// Read noisy from previous calibration
	217	AliCDBEntry* cdbEntry = GetFromOCDB("Calib", "SPDNoisy");
	218	TObjArray* spdEntryNoisy;
53ae21ce	219	if(cdbEntry) {
6727e2db	220	spdEntryNoisy = (TObjArray*)cdbEntry->GetObject();
6727e2db	221	if(!spdEntryNoisy) return 1;
03fc6773	222	}
53ae21ce	223	else {
53ae21ce	224	Log("Old calibration not found in database. This is required for further processing.");
0c0426a2	225	return 1;
0c0426a2	226	}
6727e2db	227	AliITSOnlineCalibrationSPDhandler* handOld = new AliITSOnlineCalibrationSPDhandler();
	228	handOld->ReadNoisyFromCalibObj(spdEntryNoisy);
	229	// Retrieve and unpack tared calibration files from FXS
4f88491f	230	TString idN = "SPD_phys_noisy";
4f88491f	231	TList* listN = GetFileSources(kDAQ,idN.Data());
6727e2db	232	if (listN) {
	233	UInt_t index = 0;
	234	while (listN->At(index)!=NULL) {
	235	TObjString* fileNameEntry = (TObjString*) listN->At(index);
4f88491f	236	TString fileName = GetFile(kDAQ, idN.Data(), fileNameEntry->GetString().Data());
	237	if (fileName.IsNull()) {
	238	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	239	return 1;
	240	}
	241	if (!RemoveIdFromList(idN.Data())) {
	242	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",idN.Data()));
	243	}
6727e2db	244	TString command = Form("tar -xf %s",fileName.Data());
	245	gSystem->Exec(command.Data());
	246	index++;
53ae21ce	247	}
6727e2db	248	}
	249	AliITSOnlineCalibrationSPDhandler* handNew = new AliITSOnlineCalibrationSPDhandler();
	250	handNew->SetFileLocation(".");
	251	handNew->ReadNoisyFromFiles();
	252	// add the new list to the old one
	253	UInt_t nrNewNoisy = handOld->AddNoisyFrom(handNew);
	254	// If new noisy pixels were found: Update calibration objects
	255	if (nrNewNoisy>0) {
53ae21ce	256	for (Int_t module=0; module<240; module++) {
4821f07d	257	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetNrBadSingle( handOld->GetNrNoisySingle(module) );
6727e2db	258	((AliITSCalibrationSPD*) spdEntryNoisy->At(module)) -> SetBadList( handOld->GetNoisyArray(module) );
53ae21ce	259	}
6727e2db	260	// Store the new calibration objects in OCDB
	261	Log("Noisy lists (phys) will be stored...");
	262	AliCDBMetaData metaData;
	263	metaData.SetBeamPeriod(0);
	264	metaData.SetResponsible("Henrik Tydesjo");
	265	metaData.SetComment("Created by SPD PreProcessor");
	266	// validity for this run only
	267	if (!Store("Calib", "SPDNoisy", spdEntryNoisy, &metaData, 0, kFALSE)) {
	268	Log("Failed to store calibration data.");
	269	return 1;
53ae21ce	270	}
6727e2db	271	Log("Database updated.");
53ae21ce	272	}
6727e2db	273	delete handNew;
53ae21ce	274
6727e2db	275	// Dead pixels:
6727e2db	276	// Retrieve and unpack tared calibration files from FXS
4f88491f	277	TString idD = "SPD_phys_dead";
	278	TList* listD = GetFileSources(kDAQ,idD.Data());
	279	UInt_t nrPhysDeadFiles = 0;
6727e2db	280	if (listD) {
	281	UInt_t index = 0;
	282	while (listD->At(index)!=NULL) {
	283	TObjString* fileNameEntry = (TObjString*) listD->At(index);
4f88491f	284	TString fileName = GetFile(kDAQ, idD.Data(), fileNameEntry->GetString().Data());
	285	if (fileName.IsNull()) {
	286	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	287	return 1;
	288	}
	289	nrPhysDeadFiles++;
	290	if (!RemoveIdFromList("SPD_phys_dead")) {
	291	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",idD.Data()));
	292	}
6727e2db	293	TString command = Form("tar -xf %s",fileName.Data());
	294	gSystem->Exec(command.Data());
	295	index++;
	296	}
53ae21ce	297	}
4f88491f	298	if (nrPhysDeadFiles==0) {
	299	Log(Form("Could not find files with id %s. Should be present for each run.",idD.Data()));
	300	return 1;
	301	}
6727e2db	302	// Create new database entries
	303	TObjArray* spdEntryDead = new TObjArray(240);
	304	spdEntryDead->SetOwner(kTRUE);
	305	for(UInt_t module=0; module<240; module++){
	306	AliITSCalibrationSPD* calObj = new AliITSCalibrationSPD();
	307	spdEntryDead->Add(calObj);
	308	}
	309	// Add dead from the copied FXS files
	310	handOld->SetFileLocation(".");
b696414b	311	handOld->ReadSilentFromFiles();
478d804c	312	for (UInt_t module=0; module<240; module++) {
478d804c	313	AliITSCalibrationSPD* calibSPD = (AliITSCalibrationSPD*) spdEntryDead->At(module);
4821f07d	314	calibSPD->SetNrBadSingle( handOld->GetNrDeadSingle(module) );
478d804c	315	calibSPD->SetBadList( handOld->GetDeadArray(module) );
	316	for (UInt_t chipIndex=0; chipIndex<5; chipIndex++) {
	317	UInt_t eq,hs,chip,col,row;
	318	AliITSRawStreamSPD::OfflineToOnline(module, chipIndex*32, 0, eq, hs, chip, col, row);
	319	if (handOld->IsSilentChip(eq,hs,chip)) {
	320	calibSPD->SetChipBad(chipIndex);
	321	}
	322	else {
	323	calibSPD->UnSetChipBad(chipIndex);
	324	}
	325	}
6727e2db	326	}
	327	delete handOld;
	328	// Store the new calibration objects in OCDB
	329	Log("Dead lists (phys) will be stored...");
	330	AliCDBMetaData metaData;
	331	metaData.SetBeamPeriod(0);
	332	metaData.SetResponsible("Henrik Tydesjo");
	333	metaData.SetComment("Created by SPD PreProcessor");
	334	// validity for this run only
	335	if (!Store("Calib", "SPDDead", spdEntryDead, &metaData, 0, kFALSE)) {
	336	Log("Failed to store calibration data.");
	337	return 1;
	338	}
	339	Log("Database updated.");
	340	delete spdEntryDead;
53ae21ce	341
03fc6773	342	}
03fc6773	343
03fc6773	344
af866a70	345
af866a70	346	// ******************************************************************************************** //
4f88491f	347	// check that there are no ids left in the list:
	348	if (fIdList.First()!=NULL) {
	349	TString logMessage = "";
	350	TListIter *iter = new TListIter(&fIdList);
	351	while (TObjString st = (TObjString)iter->Next()) {
	352	logMessage.Append(st->GetString());
	353	logMessage.Append(" ");
	354	}
	355	delete iter;
	356	Log(Form("Files with the following ids were never retrieved: %s.",logMessage.Data()));
	357	return 1;
	358	}
4f88491f	359	fIdList.Clear();
4f88491f	360
af866a70	361
53ae21ce	362	return 0; // 0 means success
03fc6773	363
fcf95fc7	364	}
4f88491f	365	//_________________________________________________________________________________________
	366	Bool_t AliITSPreprocessorSPD::RemoveIdFromList(const Char_t *id) {
	367	// removes id from the list of ids
	368	Bool_t found = kFALSE;
	369	TListIter *iter = new TListIter(&fIdList);
	370	while (TObjString st = (TObjString)iter->Next()) {
	371	if (st->GetString().CompareTo(id)==0) {
	372	fIdList.Remove(st);
	373	found = kTRUE;
	374	break;
	375	}
	376	}
	377	delete iter;
	378	return found;
	379	}
	380	//_________________________________________________________________________________________
af866a70	381	Bool_t AliITSPreprocessorSPD::StoreRefForIdStartingWith(const Char_t *idStart) {
	382	// Store the standalone container files as reference data (0 or 1 file for each equipment)
	383	// idStart is the first part of the id string (which also should contain the actual eq)
	384	for (UInt_t eq=0; eq<20; eq++) {
	385	TString id = Form("%s_%d",idStart,eq);
	386	TList* list = GetFileSources(kDAQ,id.Data()); // (the id should be unique, so always 1 file)
	387	if (list) {
	388	TObjString* fileNameEntry = (TObjString*) list->First();
	389	if (fileNameEntry!=NULL) {
	390	TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
	391	if (fileName.IsNull()) {
	392	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	393	return kFALSE;
	394	}
	395	if (!RemoveIdFromList(id.Data())) {
	396	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id.Data()));
	397	}
	398	TString refCAT = Form("%s_eq_%d",idStart,eq);
	399	if (!StoreReferenceFile(fileName.Data(),refCAT.Data())) {
	400	Log(Form("Failed to store reference file %s.",fileName.Data()));
	401	return kFALSE;
	402	}
	403	}
	404	}
	405	}
	406	return kTRUE;
	407	}
	408	//_________________________________________________________________________________________
4f88491f	409	Bool_t AliITSPreprocessorSPD::StoreRefFromTarForId(const Char_t *id) {
	410	// store reference files from tar file for the id given (this is just to not duplicate code)
	411	TList* list = GetFileSources(kDAQ,id);
	412	if (list) {
	413	UInt_t index = 0;
	414	while (list->At(index)!=NULL) {
	415	TObjString* fileNameEntry = (TObjString*) list->At(index);
	416	TString fileName = GetFile(kDAQ, id, fileNameEntry->GetString().Data());
	417	if (fileName.IsNull()) {
	418	Log(Form("GetFile failed to retrieve file %s.",fileNameEntry->GetString().Data()));
	419	return kFALSE;
	420	}
	421	if (!RemoveIdFromList(id)) {
	422	Log(Form("Warning: Retrieved file with id %s, that was not in the id list!",id));
	423	}
	424	// get the file names from the tar file
	425	// TString pwd = gSystem->pwd();
	426	// TString tempFileName = Form("%s/tempTar.txt",pwd.Data());
	427	TString tempFileName = "tempTar.txt";
	428	TString command = Form("tar -tf %s > %s",fileName.Data(),tempFileName.Data());
	429	gSystem->Exec(command.Data());
	430	TList fList;
	431	ifstream tempFile;
	432	tempFile.open(tempFileName.Data(), ifstream::in);
	433	if (tempFile.fail()) {
	434	Log(Form("Could not open file (%s) for reading.",tempFileName.Data()));
	435	return kFALSE;
	436	}
	437	else {
	438	while(1) {
	439	Char_t fileN[100];
	440	tempFile >> fileN;
	441	if (tempFile.eof()) break;
	442	fList.AddLast(new TObjString(fileN));
	443	}
	444	}
	445	// close and remove temp file
	446	tempFile.close();
	447	command = Form("rm -f %s",tempFileName.Data());
	448	gSystem->Exec(command.Data());
	449	// unpack
	450	command = Form("tar -xf %s",fileName.Data());
	451	gSystem->Exec(command.Data());
	452	// store each file
	453	UInt_t index2 = 0;
	454	while (fList.At(index2)!=NULL) {
8f02666b	455	TString eqFileName = ((TObjString*)fList.At(index2))->GetString();
4f88491f	456	// get eq id
8f02666b	457	TString eqStr = eqFileName.Data();
4f88491f	458	UInt_t len = eqStr.Length();
	459	eqStr.Replace(0,len-7,"",0);
	460	eqStr.ReplaceAll("_",1,"",0);
	461	eqStr.ReplaceAll(".root",5,"",0);
	462	Int_t eqId = eqStr.Atoi();
	463	if (eqId>=0 && eqId<20) {
	464	TString refCAT = Form("%s_eq_%d",id,eqId);
8f02666b	465	if (!StoreReferenceFile(eqFileName.Data(),refCAT.Data())) {
8f02666b	466	Log(Form("Failed to store reference file %s.",eqFileName.Data()));
4f88491f	467	return kFALSE;
	468	}
	469	}
	470	else {
8f02666b	471	Log(Form("Eq ID %d out of bounds for file %s",eqId,eqFileName.Data()));
4f88491f	472	fList.Clear();
	473	return kFALSE;
	474	}
	475	index2++;
	476	}
	477	fList.Clear();
	478	index++;
	479	}
	480	}
	481	return kTRUE;
	482	}