3 // Author: Anders Vestbo <mailto:vestbo$fi.uib.no>
4 //*-- Copyright © ALICE HLT Group
5 //_____________________________________________________________
9 // Compression class which performs Arithmetic Coding of the quantized residuals.
10 // The implemented algorithm is inspired by the examples in The Data Compression Book
11 // by Nelson & Gailly.
17 #include "AliHLTStandardIncludes.h"
18 #include "AliHLTTrackArray.h"
19 #include "AliHLTModelTrack.h"
20 #include "AliHLTTransform.h"
21 #include "AliHLTMemHandler.h"
22 #include "AliHLTCompress.h"
23 #include "AliHLTDataCompressorHelper.h"
24 #include "AliHLTCompressAC.h"
27 ClassImp(AliHLTCompressAC)
29 AliHLTCompressAC::AliHLTCompressAC()
31 // default constructor
42 AliHLTCompressAC::AliHLTCompressAC(Int_t slice,Int_t patch,Char_t *path,Bool_t writeshape,Int_t event) :
43 AliHLTCompress(slice,patch,path,writeshape,event)
56 AliHLTCompressAC::~AliHLTCompressAC()
62 void AliHLTCompressAC::ClearArrays()
72 void AliHLTCompressAC::BuildModel(BIT_FILE *output)
74 //Build the model from the input data, i.e. probability distributions of the quantized residuals.
79 UInt_t nmax=10000,qres;
80 UInt_t * temp = new UInt_t[nmax];
81 memset(&temp[0],0,nmax*sizeof(UInt_t));
83 AliHLTTrackArray *tracks = GetTracks();
84 for(Int_t t=0; t<tracks->GetNTracks(); t++)
86 AliHLTModelTrack *track = (AliHLTModelTrack*)tracks->GetCheckedTrack(t);
88 for(Int_t padrow=0; padrow<AliHLTTransform::GetNRows(); padrow++)
90 if(!track->IsPresent(padrow)) continue;
91 qres = abs((Int_t)rint(track->GetClusterModel(padrow)->fDPad));
94 cerr<<"AliHLTCompressAC::BuildModel() : Residual values seems way too big!"<<endl;
100 qres = abs((Int_t)rint(track->GetClusterModel(padrow)->fDTime));
108 fCount = new UChar_t[fMax+1];
110 //Find the highest counts in order to do scaling:
112 for(i=0; i<=fMax; i++)
114 if(temp[i] > maxCount)
118 //Perform the scaling
119 UInt_t scale,total=1;
120 scale = maxCount / 256 + 1;
121 for(i=0; i<=fMax; i++)
123 fCount[i] = (UChar_t)(temp[i]/scale);
124 if(fCount[i]==0 && temp[i]!=0)
126 total += (UInt_t)fCount[i];
128 if(total > (32767 - 256))
130 else if(total > 16383)
135 for(i=0; i<=fMax; i++)
138 cout<<"Writing "<<sizeof(UChar_t)*fMax+1<<" bytes with model information to compressed file"<<endl;
139 fwrite(&fMax,sizeof(UShort_t),1,output->file);
140 fwrite(fCount,sizeof(UChar_t),fMax+1,output->file);
146 void AliHLTCompressAC::RebuildModel(BIT_FILE *input)
148 //Rebuild the model from the counts written to the beginning of the compressed file.
151 fread(&fMax,sizeof(UShort_t),1,input->file);
152 fCount = new UChar_t[fMax+1];
153 fread(fCount,sizeof(UChar_t),fMax+1,input->file);
157 void AliHLTCompressAC::FillTotals()
159 //Fill the array of totals, which is actually the model being used during encoding/decoding.
161 cerr<<"AliHLTCompressAC::FillTotals : max value is zero!"<<endl;
163 fTotals = new UInt_t[fMax+3];//up to max, and one reserved for endofstream symbol
167 for(i=0; i<=fMax; i++)
169 fTotals[i+1] = fTotals[i] + fCount[i];
171 fTotals[fMax+2] = fTotals[fMax+1]+1;//Used for the scale
174 void AliHLTCompressAC::PrintTotals() const
177 cout<<"Totals:"<<endl;
178 for(UInt_t i=0; i<=fMax; i++)
180 cout<<"Totals "<<i<<" "<<fTotals[i]<<" count "<<(int)fCount[i]<<endl;
184 void AliHLTCompressAC::InitEncoder()
192 void AliHLTCompressAC::InitDecoder(BIT_FILE *input)
196 for(Int_t i=0; i<16; i++)
199 fCode += InputBit(input);
205 void AliHLTCompressAC::ConvertIntToSymbol(Int_t value)
207 // converst integer to symbol
208 UInt_t range = fHigh - fLow + 1;
209 fHigh = fLow + (UShort_t)((range*fTotals[value+1])/fTotals[fMax+2] - 1);
210 fLow = fLow + (UShort_t)((range*fTotals[value])/fTotals[fMax+2]);
213 UInt_t AliHLTCompressAC::ConvertSymbolToInt()
215 // converts symbol to integer
216 UInt_t range = (UInt_t)(fHigh-fLow) + 1;
217 UShort_t count = (UShort_t)((((UInt_t)(fCode-fLow)+1)*fTotals[fMax+2] - 1)/range);
219 while(count < fTotals[j])
225 void AliHLTCompressAC::EncodeSymbol(BIT_FILE *output)
230 if( (fHigh & 0x8000) == (fLow & 0x8000) )
232 OutputBit(output,fHigh & 0x8000);
233 while(fUnderflowBits > 0)
235 OutputBit(output,~fHigh & 0x8000);
239 else if( (fLow & 0x4000) && !(fHigh & 0x4000) )
253 void AliHLTCompressAC::RemoveSymbolFromStream(BIT_FILE *input,Int_t j)
255 // remves symbol fro stream
256 UInt_t range = (UInt_t)(fHigh-fLow)+1;
257 fHigh = fLow + (UShort_t)((range*fTotals[j+1])/fTotals[fMax+2]-1);
258 fLow = fLow + (UShort_t)((range*fTotals[j])/fTotals[fMax+2]);
261 if( (fHigh & 0x8000)==(fLow & 0x8000) )
263 else if((fLow & 0x4000) == 0x4000 && (fHigh & 0x4000)==0)
275 fCode += InputBit(input);
279 void AliHLTCompressAC::FlushEncoder(BIT_FILE *output)
282 OutputBit(output,fLow & 0x4000);
284 while(fUnderflowBits-- > 0)
285 OutputBit(output,~fLow & 0x4000);
290 Bool_t AliHLTCompressAC::CompressFile()
295 sprintf(fname,"%s/comp/tracks_ac_%d_%d.raw",fPath,fSlice,fPatch);
297 sprintf(fname,"%s/comp/tracks_ac_%d_%d_%d.raw",fPath,fEvent,fSlice,fPatch);
298 BIT_FILE *output = OpenOutputBitFile(fname);
301 sprintf(fname,"%s/comp/tracks_m_%d_%d.raw",fPath,fSlice,fPatch);
303 sprintf(fname,"%s/comp/tracks_m_%d_%d_%d.raw",fPath,fEvent,fSlice,fPatch);
305 FILE *input = fopen(fname,"r");
308 cerr<<"AliHLTCompressAC::CompressFileAC() : Error opening file: "<<fname<<endl;
314 AliHLTTrackModel track;
315 Int_t temp,power,i,j;
317 fseek(input,0,SEEK_END);
318 UInt_t size = ftell(input);
320 Int_t trackcount = size/(sizeof(AliHLTTrackModel) + sizeof(AliHLTClusterModel)*AliHLTTransform::GetNRows(fPatch));
322 //Write the number of tracks in the beginning of stream.
323 fwrite(&trackcount,sizeof(Int_t),1,output->file);
325 AliHLTClusterModel **clusters = new AliHLTClusterModel*[trackcount];
326 Int_t *clustercount = new Int_t[trackcount];
329 //Read all the tracks from input file, and write them all to the outputfile.
330 //Store the clusters in memory for later encoding and storing.
333 if(fread(&track,sizeof(AliHLTTrackModel),1,input)!=1) break;
334 fwrite(&track,sizeof(AliHLTTrackModel),1,output->file);
336 clusters[i] = new AliHLTClusterModel[AliHLTTransform::GetNRows()];
339 //Read in the clusters:
340 fread(clusters[i],sizeof(AliHLTClusterModel),AliHLTTransform::GetNRows(fPatch),input);
345 cerr<<"AliHLTCompressAC::CompressFile : Mismatching file size and trackcount "<<i<<" "<<trackcount<<endl;
350 //Write all the fixed size variables of the clusters:
351 for(i=0; i<trackcount; i++)
353 Int_t origslice=-1,slice;
354 for(j=0; j<AliHLTTransform::GetNRows(fPatch); j++)
356 temp = (Int_t)clusters[i][j].fPresent;
357 OutputBit(output,temp);
360 if(clusters[i][j].fSlice<0 || clusters[i][j].fSlice>35)
362 cerr<<"AliHLTDataCompress::CompressFile : Fucked up slice number :"<<clusters[i][j].fSlice<<endl;
366 //Write slice number of first point
367 if(clustercount[i]==0)
369 origslice = clusters[i][j].fSlice;
370 OutputBits(output,origslice,6); //Need 6 bits to encode slice number
374 slice = clusters[i][j].fSlice;
375 if(slice == origslice)
377 OutputBit(output,0); //No change of slice
382 if(abs(slice - origslice)==1)
384 if(slice > origslice)
391 if( (slice == 0 && origslice == 17) || (slice == 18 && origslice == 35) )
393 else if( (slice == 17 && origslice == 0) || (slice == 35 && origslice == 18) )
400 //Write charge information:
401 temp = (Int_t)clusters[i][j].fDCharge;
402 power = 1<<(AliHLTDataCompressorHelper::GetNChargeBits());
407 OutputBits(output,abs(temp),(AliHLTDataCompressorHelper::GetNChargeBits()));
409 //Write sign information of the residuals:
410 temp = (Int_t)rint(clusters[i][j].fDTime);
415 temp = (Int_t)rint(clusters[i][j].fDPad);
421 //Write shape information if requested:
424 temp = (Int_t)rint(clusters[i][j].fDSigmaY);
429 power = 1<<(AliHLTDataCompressorHelper::GetNShapeBits()-1);
430 if(abs(temp) >= power)
434 OutputBits(output,abs(temp),(AliHLTDataCompressorHelper::GetNShapeBits()-1));
436 temp = (Int_t)rint(clusters[i][j].fDSigmaZ);
441 power = 1<<(AliHLTDataCompressorHelper::GetNShapeBits()-1);
442 if(abs(temp) >= power)
446 OutputBits(output,abs(temp),(AliHLTDataCompressorHelper::GetNShapeBits()-1));
454 //Start the arithmetic coding of the residuals.
455 //All the residuals (both pad and time) are coded in one go,
456 //i.e. for all tracks and clusters in the input file.
457 for(i=0; i<trackcount; i++)
461 for(j=0; j<AliHLTTransform::GetNRows(fPatch); j++)
463 if(!clusters[i][j].fPresent) continue;
464 temp = abs((Int_t)rint(clusters[i][j].fDTime));
465 ConvertIntToSymbol(temp);
466 EncodeSymbol(output);
468 temp = abs((Int_t)rint(clusters[i][j].fDPad));
469 ConvertIntToSymbol(temp);
470 EncodeSymbol(output);
473 if(counter != clustercount[i])
475 cerr<<"AliHLTCompressAC::CompressFile : Mismatching clustercount "<<counter<<" "<<clustercount[i]<<endl;
481 ConvertIntToSymbol(fMax+1);//End of stream symbol
482 EncodeSymbol(output);
483 FlushEncoder(output);
484 OutputBits(output,0,16);
486 for(i=0; i<trackcount; i++)
487 delete [] clusters[i];
489 delete [] clustercount;
492 CloseOutputBitFile(output);
497 Bool_t AliHLTCompressAC::ExpandFile()
502 sprintf(fname,"%s/comp/tracks_ac_%d_%d.raw",fPath,fSlice,fPatch);
504 sprintf(fname,"%s/comp/tracks_ac_%d_%d_%d.raw",fPath,fEvent,fSlice,fPatch);
505 BIT_FILE *input = OpenInputBitFile(fname);
508 sprintf(fname,"%s/comp/tracks_u_%d_%d.raw",fPath,fSlice,fPatch);
510 sprintf(fname,"%s/comp/tracks_u_%d_%d_%d.raw",fPath,fEvent,fSlice,fPatch);
511 FILE *output = fopen(fname,"w");
514 cerr<<"AliHLTCompress::ExpandFile() : Error opening file: "<<fname<<endl;
521 fread(&trackcount,sizeof(Int_t),1,input->file);
523 AliHLTTrackModel *trackmodels = new AliHLTTrackModel[trackcount];
524 AliHLTClusterModel **clusters = new AliHLTClusterModel*[trackcount];
525 Int_t *clustercount = new Int_t[trackcount];
527 fread(trackmodels,sizeof(AliHLTTrackModel),trackcount,input->file);
529 for(i=0; i<trackcount; i++)
531 clusters[i] = new AliHLTClusterModel[AliHLTTransform::GetNRows(fPatch)];
534 //Read the fixed size variables:
537 for(j=0; j<AliHLTTransform::GetNRows(fPatch); j++)
540 temp = InputBit(input);
543 clusters[i][j].fPresent=kFALSE;
546 clusters[i][j].fPresent=kTRUE;
548 //Read slice information
549 if(clustercount[i]==0)
551 temp = InputBits(input,6);
552 clusters[i][j].fSlice = temp;
557 temp = InputBit(input);
559 clusters[i][j].fSlice = origslice;
562 temp = InputBit(input);
567 else if(origslice == 35)
576 else if(origslice == 18)
581 if(origslice < 0 || origslice > 35)
583 cerr<<"AliHLTCompressAC::ExpandFile : Bad slice number "<<temp<<endl;
586 clusters[i][j].fSlice = origslice;
590 //Read charge information:
591 temp=InputBits(input,(AliHLTDataCompressorHelper::GetNChargeBits()));
592 clusters[i][j].fDCharge = temp;
594 //Read sign information of the residuals:
595 sign=InputBit(input);
597 clusters[i][j].fDTime = -1;
599 clusters[i][j].fDTime = 1;
600 sign=InputBit(input);
602 clusters[i][j].fDPad = -1;
604 clusters[i][j].fDPad = 1;
606 //Read shape information if requested
609 sign = InputBit(input);
610 temp = InputBits(input,(AliHLTDataCompressorHelper::GetNShapeBits()-1));
613 clusters[i][j].fDSigmaY = temp;
615 sign = InputBit(input);
616 temp = InputBits(input,(AliHLTDataCompressorHelper::GetNShapeBits()-1));
619 clusters[i][j].fDSigmaZ = temp;
628 for(i=0; i<trackcount; i++)
631 for(j=0; j<AliHLTTransform::GetNRows(fPatch); j++)
633 if(!clusters[i][j].fPresent) continue;
635 temp = ConvertSymbolToInt();
636 RemoveSymbolFromStream(input,temp);
637 clusters[i][j].fDTime *= temp;
639 temp = ConvertSymbolToInt();
640 RemoveSymbolFromStream(input,temp);
641 clusters[i][j].fDPad *= temp;
645 if(count != clustercount[i])
647 cerr<<"AliHLTCompressAC::ExpandFile : Mismatching clustercount "<<count<<" "<<clustercount[i]<<endl;
652 //Now there should be a endofstream indicator, if not something went wrong during encoding/decoding.
653 temp = ConvertSymbolToInt();
654 if((UShort_t)temp != fMax + 1)
655 cerr<<"AliHLTCompressAC::ExpandFile : Missing the endofstream indicator!"<<endl;
657 CloseInputBitFile(input);
659 //Write everything to the uncompressed outfile:
660 for(i=0; i<trackcount; i++)
662 fwrite(&trackmodels[i],sizeof(AliHLTTrackModel),1,output);
663 fwrite(clusters[i],AliHLTTransform::GetNRows(fPatch)*sizeof(AliHLTClusterModel),1,output);
668 for(i=0; i<trackcount; i++)
669 delete [] clusters[i];
671 delete [] trackmodels;
672 delete [] clustercount;
677 void AliHLTCompressAC::PrintCompRatio(ofstream *outfile)
679 // pristc compression ratio
680 AliHLTMemHandler *mem = new AliHLTMemHandler();
682 UInt_t remainSize=0,digitSize=0;
683 for(Int_t i=0; i<36; i++)
686 sprintf(fname,"%s/comp/remains_%d_%d.raw",fPath,i,-1);
688 sprintf(fname,"%s/comp/remains_%d_%d_%d.raw",fPath,fEvent,i,-1);
689 mem->SetBinaryInput(fname);
690 remainSize += mem->GetFileSize();
691 mem->CloseBinaryInput();
693 sprintf(fname,"%s/binaries/digits_c8_%d_%d_%d.raw",fPath,fEvent,i,-1);
694 mem->SetBinaryInput(fname);
695 digitSize += mem->GetFileSize();
696 mem->CloseBinaryInput();
701 sprintf(fname,"%s/comp/tracks_ac_%d_%d.raw",fPath,fSlice,fPatch);
703 sprintf(fname,"%s/comp/tracks_ac_%d_%d_%d.raw",fPath,fEvent,fSlice,fPatch);
705 mem->SetBinaryInput(fname);
706 UInt_t compressSize = mem->GetFileSize();
707 mem->CloseBinaryInput();
711 cerr<<"AliHLTCompress::PrintCompRatio : Zero digit size, not able to obtain comp. ratios!"<<endl;
715 Float_t compratio = (Float_t)(compressSize + remainSize)/(Float_t)digitSize;
717 Int_t trackSize = GetEntropy(entropy[0],entropy[1],entropy[2])*sizeof(AliHLTTrackModel);
720 ofstream &out = *outfile;
721 out<<compressSize<<' '<<remainSize<<' '<<digitSize<<' '<<trackSize<<' '<<entropy[0]<<' '<<entropy[1]<<endl;
724 cout<<"=========================================="<<endl;
725 cout<<"Original digits size : "<<digitSize/1000<<" kByte ( 100 % )"<<endl;
726 cout<<"Compressed file size : "<<compressSize/1000<<" kByte ( "<<(Float_t)compressSize*100/(Float_t)digitSize<<" % )"<<endl;
727 cout<<"Remaining file size : "<<remainSize/1000<<" kByte ( "<<(Float_t)remainSize*100/(Float_t)digitSize<<" % )"<<endl;
728 cout<<"Relative track size : "<<trackSize/1000<<" kByte ( "<<(Float_t)trackSize*100/(Float_t)compressSize<<" % )"<<endl;
729 cout<<"Relative cluster size: "<<(compressSize-trackSize)/1000<<" kByte ( "<<(Float_t)(compressSize-trackSize)*100/(Float_t)compressSize<<" % )"<<endl;
730 cout<<"---------------------- "<<endl;
731 cout<<"Compression ratio : "<<compratio*100<<" %"<<endl;
732 cout<<"=========================================="<<endl;
733 cout<<"Entropy of residual and charge : "<<entropy[0]<<" "<<entropy[1]<<" "<<entropy[2]<<endl;