4 #ifndef ALIHLTDATABUFFER_H
5 #define ALIHLTDATABUFFER_H
6 //* This file is property of and copyright by the ALICE HLT Project *
7 //* ALICE Experiment at CERN, All rights reserved. *
8 //* See cxx source for full Copyright notice *
10 // @file AliHLTDataBuffer.h
11 // @author Matthias Richter
13 // @brief Handling of Data Buffers for HLT components.
14 // @note The class is used in Offline (AliRoot) context
16 // see below for class documentation
18 // refer to README to build package
20 // visit http://web.ift.uib.no/~kjeks/doc/alice-hlt
24 #include "AliHLTLogging.h"
25 #include "AliHLTDataTypes.h"
26 #include "AliHLTComponent.h"
28 class AliHLTConsumerDescriptor;
31 /** list of AliHLTConsumerDescriptor pointers */
32 typedef vector<AliHLTConsumerDescriptor*> AliHLTConsumerDescriptorPList;
34 typedef AliHLTUInt8_t* AliHLTUInt8Pointer_t;
37 * @class AliHLTDataBuffer
38 * @brief Handling of data buffers for the HLT.
40 * The class provides handling of data buffers for HLT tasks. Each task gets
41 * its own Data Buffer instance. The buffer is grouped into different data
42 * segments according to the output of the component.<br>
43 * The Data Buffer keeps control over the data requests of the 'child'
44 * components. Each component can subscribe to a certain segment of the data
45 * buffer. It's state is then changed from 'reserved' to 'active'. After the
46 * data processing, the component has to release the segment and it's state is
47 * set to 'processed'. If all components have requested and released their data,
48 * the Raw Buffer is released and pushed back in the list of available buffers.
50 * @note This class is only used for the @ref alihlt_system.
52 * @ingroup alihlt_system
54 class AliHLTDataBuffer : public TObject, public AliHLTLogging
57 //////////////////////////////////////////////////////////////////////////////
58 // constructors and destructors
60 /* standard constructor
64 virtual ~AliHLTDataBuffer();
66 //////////////////////////////////////////////////////////////////////////////
70 * Add component to the list of consumers
71 * @param pConsumer - a consumer of type AliHLTComponent
73 int SetConsumer(AliHLTComponent* pConsumer);
75 //////////////////////////////////////////////////////////////////////////////
76 // component to component communication
79 * Determine the number of matching data blocks for the component and a
80 * consumer component. <br>
81 * The first approach will support only one output data type for processing
83 * @param pConsumer the component which subscribes to the buffer
84 * @param tgtList (optional) the list to receive the data types
85 * @return: number of data blocks which match the input data types
86 * of the consumer, neg. error code if failed <br>
87 * -EINVAL invalid parameter <br>
89 int FindMatchingDataBlocks(const AliHLTComponent* pConsumer,
90 AliHLTComponentDataTypeList* tgtList=NULL);
93 * Subscribe to a segment of the data buffer.
94 * The function prepares the block descriptor for subsequent use with the
95 * AliHLTComponent::ProcessEvent method, the method can prepare several block
96 * descriptors up to the array size specified by iArraySize. The return value
97 * is independent from the array size the number of block descriptors which
98 * would have been prepared if there was enough space in the array<br>
99 * The method is used by the consumer component.
100 * @param pConsumer the component which subscribes to the buffer
101 * @param blockDescList block descriptor vector to be filled
102 * @return: number of matching data blocks, neg. error code if failed<br>
103 * -EACCESS the consumer state can't be changed (activated)
104 * -EBADF unresolved data segments <br>
105 * -ENOENT consumer component not found <br>
106 * -ENODATA data buffer does not have raw data <br>
107 * -EINVAL invalid parameter <br>
109 int Subscribe(const AliHLTComponent* pConsumer,
110 AliHLTComponentBlockDataList& blockDescList);
113 * Release an instance of the data buffer.
114 * Resets the variables of the block descriptor.
115 * If all buffer segments are released, the Data Buffer is reseted
116 * and the Raw Buffer released.<br>
117 * The method is used by the consumer component.
118 * @param pBlockDesc descriptor of the data segment
119 * @param pConsumer the component which subscribes to the buffer
120 * @param pOwnerTask task owning this buffer
121 * @return: >0 if success, negative error code if failed <br>
122 * -EACCESS the consumer state can not be changed (de-activated)
123 * -ENOENT consumer has not subscribed to the buffer <br>
124 * -EINVAL invalid parameter <br>
126 int Release(AliHLTComponentBlockData* pBlockDesc, const AliHLTComponent* pConsumer,
127 const AliHLTTask* pOwnerTask);
130 * Release a forwarded data block.
132 int ReleaseForwardedBlock(AliHLTComponentBlockData* pBlockDesc,
133 const AliHLTTask* pOwnerTask);
136 * Register an input data block for forwarding.
137 * Consumer of this data buffer subscribe to forwarded data blocks in te same way.
138 * Forwarded data blocks are released when the last consumer has released the
140 * @param pSrcTask original source task of the data block
141 * @param pBlockDesc descriptor of the data segment
143 int Forward(AliHLTTask* pSrcTask, AliHLTComponentBlockData* pBlockDesc);
146 * Get a target buffer of minimum size iMinSize.
147 * The method is used by the component which owns the Data Buffer to
148 * allocate a buffer for the data it is going to produce.
149 * @param iMinSize minumum size of the requested buffer
150 * @return: pointer to target buffer if
152 AliHLTUInt8_t* GetTargetBuffer(int iMinSize);
155 * Set the segments for the data buffer.
156 * This is usually done after the component has written the data to the buffer,
157 * which was requested by the @ref GetTargetBuffer method. The component might
158 * produce different types of data, for each type a segment has to be defined
159 * which describes the data inside the buffer.<br>
160 * The @ref AliHLTComponentBlockData segment descriptor comes directly from
161 * the @ref AliHLTComponent::ProcessEvent method.
162 * @param pTgt the target buffer which the segments refer to
163 * @param arraySegments the output block descriptors of the component
164 * @param iSize size of the array
166 int SetSegments(AliHLTUInt8_t* pTgt, AliHLTComponentBlockData* arraySegments, int iSize);
169 * Check if the data buffer is empty.
170 * @return 1 if empty, 0 if not
175 * Get the total and maximum size of the buffer.
176 * Lets see if this is needed later
178 //int GetTotalSize();
181 * Get the number of segments including the forwarded data blocks.
182 * @return number of segments
184 int GetNofSegments() const;
187 * Get the total number of consumers.
188 * This gives the number of consumers regardless of their state.
189 * @return number of consumers
191 int GetNofConsumers() const;
194 * Get the number of consumers which still need to be processed during
196 * @return number of consumers
198 int GetNofPendingConsumers() const;
201 * Get the number of consumers currently under processing.
202 * @return number of active consumers
204 int GetNofActiveConsumers() const;
207 * Check if a consumer is already in the list
208 * @param pConsumer pointer to consumer component
209 * @param bAllLists search in all lists if 1
210 * search only in fConsumer list if 0
211 * @return 1 if found, 0 if not
213 int FindConsumer(const AliHLTComponent* pConsumer, int bAllLists=1);
216 * Public method to reset the buffer.
217 * Eventually with some additional checks. In normal operation,
218 * an external reset should not be necessary.
223 * Print info about the buffer
225 virtual void Print(const char* option) const;
228 * Set local logging level
229 * logging filter for individual object
231 void SetLocalLoggingLevel(AliHLTComponentLogSeverity level)
232 {fgLogging.SetLocalLoggingLevel(level); AliHLTLogging::SetLocalLoggingLevel(level);}
235 * Print summary of the global buffer management.
237 static int PrintStatistics();
240 * Set the global event count.
241 * The event count is deployed to find buffers which have not been used
242 * for a while. In such a case to policy to find an appropriate buffer is
245 static int SetGlobalEventCount(AliHLTUInt32_t eventCount) {fgEventCount=eventCount; return 0;}
248 * @class AliHLTDataSegment
249 * @brief Descriptor of a data segment within the buffer.
251 class AliHLTDataSegment {
252 friend class AliHLTDataBuffer; // TODO: implement some getters/setters
256 fDataType(kAliHLTVoidDataType),
264 AliHLTDataSegment(AliHLTUInt8_t* ptr, AliHLTUInt32_t offset, AliHLTUInt32_t size)
266 fDataType(kAliHLTVoidDataType),
268 fSegmentOffset(offset),
274 AliHLTDataSegment(void* ptr, AliHLTUInt32_t offset, AliHLTUInt32_t size)
276 fDataType(kAliHLTVoidDataType),
277 fPtr(reinterpret_cast<AliHLTUInt8_t*>(ptr)),
278 fSegmentOffset(offset),
284 AliHLTDataSegment(void* ptr, AliHLTUInt32_t offset, AliHLTUInt32_t size, AliHLTComponentDataType dt, AliHLTUInt32_t spec)
287 fPtr(reinterpret_cast<AliHLTUInt8_t*>(ptr)),
288 fSegmentOffset(offset),
294 AliHLTDataSegment(const AliHLTDataSegment& src)
296 fDataType(src.fDataType),
298 fSegmentOffset(src.fSegmentOffset),
299 fSegmentSize(src.fSegmentSize),
300 fSpecification(src.fSpecification)
302 // AliHLTDataSegment just stores external pointers and properties
305 AliHLTDataSegment& operator=(const AliHLTDataSegment& src)
307 // AliHLTDataSegment just stores external pointers and properties
308 if (this==&src) return *this;
309 fDataType=src.fDataType;
311 fSegmentOffset=src.fSegmentOffset;
312 fSegmentSize=src.fSegmentSize;
313 fSpecification=src.fSpecification;
317 virtual ~AliHLTDataSegment() {}
319 AliHLTUInt8_t* GetPtr() const {return (AliHLTUInt8_t*)*this;}
321 AliHLTUInt32_t GetSize() const {return fSegmentSize;}
323 int operator==(const AliHLTDataSegment& seg) const
325 return (fPtr+fSegmentOffset==seg.fPtr+seg.fSegmentOffset) && (fSegmentSize==seg.fSegmentSize);
327 operator AliHLTUInt8_t*() const {return fPtr+fSegmentOffset;}
329 virtual void Print(const char* option) const;
332 /** the data type of this segment */
333 AliHLTComponentDataType fDataType; // see above
334 /** pointer to the buffer */
335 AliHLTUInt8Pointer_t fPtr; //!transient
336 /** offset in byte within the data buffer */
337 AliHLTUInt32_t fSegmentOffset; // see above
338 /** size of the actual content */
339 AliHLTUInt32_t fSegmentSize; // see above
340 /** data specification */
341 AliHLTUInt32_t fSpecification; // see above
346 * @class AliHLTForwardedDataSegment
347 * @brief Descriptor of a forwarded data segment.
348 * Contains in addition information about the parent of this forwarded
349 * block and the original data type and specification
351 class AliHLTForwardedDataSegment : public AliHLTDataSegment {
352 friend class AliHLTDataBuffer; // TODO: implement some getters/setters
354 AliHLTForwardedDataSegment()
355 : AliHLTDataSegment()
361 AliHLTForwardedDataSegment(AliHLTDataSegment& mySegment, AliHLTDataSegment& parentSegment, AliHLTTask* parentTask)
362 : AliHLTDataSegment(mySegment)
363 , fParentSegment(parentSegment)
364 , fParentTask(parentTask)
368 AliHLTForwardedDataSegment(const AliHLTForwardedDataSegment& src)
369 : AliHLTDataSegment(src),
370 fParentSegment(src.fParentSegment),
371 fParentTask(src.fParentTask)
373 // AliHLTForwardedDataSegment just stores external pointers and properties
376 AliHLTForwardedDataSegment& operator=(const AliHLTForwardedDataSegment& src)
378 // AliHLTForwardedDataSegment just stores external pointers and properties
379 AliHLTDataSegment::operator=(src);
380 fParentSegment=src.fParentSegment;
381 fParentTask=src.fParentTask;
385 virtual ~AliHLTForwardedDataSegment() {}
387 virtual void Print(const char* option) const;
390 /// description of the original segment
391 AliHLTDataSegment fParentSegment; // see above
393 AliHLTTask* fParentTask; //!transient
396 class AliHLTRawBuffer;
397 typedef vector<AliHLTRawBuffer*> AliHLTRawBufferPList;
400 * @class AliHLTRawPage
401 * Memory allocation is organized in pages of a fixed size. Within a
402 * page, AliHLTRawBuffer chunks are created.
404 class AliHLTRawPage : public AliHLTLogging {
406 /** standard constructor */
407 AliHLTRawPage() : fSize(0), fPtr(NULL), fFreeBuffers(), fUsedBuffers() {}
409 AliHLTRawPage(AliHLTUInt32_t pagesize);
411 virtual ~AliHLTRawPage();
413 /** alloc a buffer of specified size from the global pages*/
414 static AliHLTRawBuffer* GlobalAlloc(AliHLTUInt32_t size, int verbosity=0);
415 /** find buffer in the global pages */
416 static AliHLTRawPage* FindPage(AliHLTRawBuffer* buffer);
417 /** cleanup the global pages */
418 static int GlobalClean();
419 /** adjust global page size */
420 static void SetGlobalPageSize(AliHLTUInt32_t size) {fgGlobalPageSize=size;}
421 /** find next page after prev, or first page */
422 static AliHLTRawPage* NextPage(const AliHLTRawPage* prev=NULL);
424 /** alloc a buffer of specified size */
425 AliHLTRawBuffer* Alloc(AliHLTUInt32_t size);
426 /** free a buffer and merge consecutive free buffers */
427 int Free(AliHLTRawBuffer* pBuffer);
428 /** set the size of a raw buffer and release the remaining part */
429 int SetSize(const AliHLTRawBuffer* pBuffer, AliHLTUInt32_t size);
430 /// check if the buffer is in this page
431 bool HasBuffer(const AliHLTRawBuffer* pBuffer);
433 AliHLTUInt32_t Size() const {return fSize;}
434 AliHLTUInt32_t Capacity() const;
435 bool IsUsed() const {return fUsedBuffers.size()>0;}
436 bool IsFragmented() const {return (fFreeBuffers.size()+fUsedBuffers.size())>1;}
439 * Print page information
441 virtual void Print(const char* option);
444 /** copy constructor prohibited */
445 AliHLTRawPage(const AliHLTRawPage&);
446 /** assignment operator prohibited */
447 AliHLTRawPage& operator=(const AliHLTRawPage&);
449 /// list of global pages
450 static vector<AliHLTDataBuffer::AliHLTRawPage*> fgGlobalPages; //! transient
451 /// pages size of global pages
452 static AliHLTUInt32_t fgGlobalPageSize; //! transient
455 AliHLTUInt32_t fSize; // see above
456 /** the memory segment */
457 AliHLTUInt8_t* fPtr; //! transient
459 /** list of free buffers */
460 AliHLTRawBufferPList fFreeBuffers; //! transient
461 /** list of used buffers */
462 AliHLTRawBufferPList fUsedBuffers; //! transient
466 * @class AliHLTRawBuffer
467 * @brief Descriptor of the raw data buffer which can host several segments.
469 class AliHLTRawBuffer {
471 /** standard constructor */
472 AliHLTRawBuffer() : fSize(0), fTotalSize(0), fExternalPtr(NULL), fPtr(NULL), fLastEventCount(0) {}
474 AliHLTRawBuffer(AliHLTUInt32_t size);
476 AliHLTRawBuffer(AliHLTUInt32_t size, AliHLTUInt8_t* buffer);
478 virtual ~AliHLTRawBuffer();
481 * Use a fraction of the buffer.
482 * @param size size in bytes to be used
483 * @return pointer to buffer
485 AliHLTUInt8_t* UseBuffer(AliHLTUInt32_t size);
488 * split a buffer at specified size
489 * only possible for buffers with external memory
491 AliHLTRawBuffer* Split(AliHLTUInt32_t size);
494 * Check whether buffer fits for a request.
495 * A buffer fits if it is at least of the requested size and at most
496 * the requested size plus a margin. The margin increases with the
497 * number of events the buffer has not been used.
498 * @param size size of the request in bytes
499 * @return 1 if buffer is big enough, 0 if not
501 int CheckSize(AliHLTUInt32_t size) const;
504 * Get used size of the buffer
506 AliHLTUInt32_t GetUsedSize() const {return fSize;}
509 * Get total size of the buffer
511 AliHLTUInt32_t GetTotalSize() const {return fTotalSize;}
514 * Get pointer of data buffer
516 AliHLTUInt8_t* GetPointer() const {return fPtr;}
519 * Write check pattern
521 int WritePattern(const char* pattern, int size);
526 int CheckPattern(const char* pattern, int size) const;
530 * Data buffer remains allocated, used size set to 0
535 * Merge buffer with succeeding buffer.
536 * Only possible if the buffers are consecutive with out any gap.
538 int Merge(const AliHLTRawBuffer& succ);
541 * Print buffer information
543 virtual void Print(const char* option) const;
545 int operator==(void* ptr) const;
546 int operator==(AliHLTUInt8_t* ptr) const {return fPtr==ptr;}
547 int operator<(void* ptr) const;
548 int operator<=(void* ptr) const;
549 int operator>(void* ptr) const;
550 int operator-(void* ptr) const;
551 int operator<(const AliHLTRawBuffer& op) const;
552 int operator<=(const AliHLTRawBuffer& op) const;
553 int operator>(const AliHLTRawBuffer& op) const;
555 operator void*() const {return fPtr;}
556 operator AliHLTUInt8_t*() const {return fPtr;}
559 /** copy constructor prohibited */
560 AliHLTRawBuffer(const AliHLTRawBuffer&);
561 /** assignment operator prohibited */
562 AliHLTRawBuffer& operator=(const AliHLTRawBuffer&);
564 /** size of the currently occupied partition of the buffer */
565 AliHLTUInt32_t fSize; // see above
566 /** total size of the buffer, including safety margin */
567 AliHLTUInt32_t fTotalSize; // see above
568 /** optional external buffer */
569 AliHLTUInt8_t* fExternalPtr; //! transient
570 /** the buffer, external or allocated */
571 AliHLTUInt8_t* fPtr; //! transient
572 /** last event count where the buffer has been used */
573 AliHLTUInt32_t fLastEventCount; //! transient
577 /** copy constructor prohibited */
578 AliHLTDataBuffer(const AliHLTDataBuffer&);
579 /** assignment operator prohibited */
580 AliHLTDataBuffer& operator=(const AliHLTDataBuffer&);
582 /* lets see if this is needed
583 AliHLTDataSegment* FindDataSegment(AliHLTComponentDataType datatype);
587 * Find those data segments which match the input types of a component.
588 * @param pConsumer the component which subscribes to the buffer
589 * @param tgtList the list to receive the data segment descriptors
590 * @return: number of data blocks which match the input data types
591 * of the consumer, neg. error code if failed <br>
592 * -EINVAL invalid parameter <br>
594 int FindMatchingDataSegments(const AliHLTComponent* pConsumer,
595 vector<AliHLTDataBuffer::AliHLTDataSegment>& tgtList);
598 // 2010-02-01 make function protected in order to be used from unit test
600 * Reset the data buffer.
601 * Removes all consumers back to the @ref fConsumers list, deletes
602 * segments and releases the Raw Buffer.
604 int ResetDataBuffer();
607 //////////////////////////////////////////////////////////////////////////////
609 // the data description
611 // the data segments within this buffer
612 vector<AliHLTDataSegment> fSegments; // see above
614 // the list of all consumers which are going to subscribe to the buffer
615 AliHLTConsumerDescriptorPList fConsumers; // see above
616 // the list of all consumers which are currently subscribed to the buffer
617 AliHLTConsumerDescriptorPList fActiveConsumers; // see above
618 // the list of all consumers which are already released for the current event
619 AliHLTConsumerDescriptorPList fReleasedConsumers; // see above
621 // the buffer instance
622 AliHLTRawBuffer* fpBuffer; //! transient
624 // flags indicating the state of the buffer
625 AliHLTUInt32_t fFlags; // see above
627 /** list of tasks with forwarded data blocks */
628 vector<AliHLTTask*> fForwardedSegmentSources; //! transient
630 /** list of forwarded block descriptors */
631 vector<AliHLTDataSegment> fForwardedSegments; //! transient
633 //////////////////////////////////////////////////////////////////////////////
634 // global buffer handling, internal use only
637 * Create a raw buffer of a certain size.
638 * The function tries to find a buffer of the given size (or a bit bigger by a
639 * certain margin @ref fgMargin) from the list of free buffers.
640 * If no buffer is available, a new one is created and added to the buffer handling.
641 * @param size min. size of the requested buffer
642 * @return pointer to raw buffer
644 static AliHLTRawBuffer* CreateRawBuffer(AliHLTUInt32_t size);
647 * Set the data size of a raw buffer after it has been filled by
650 int SetRawBufferDataSize(AliHLTRawBuffer* pBuffer, AliHLTUInt32_t size) const;
653 * Mark a buffer as free.
654 * After the Data Buffer has finnished using the raw buffer, it is released
655 * and added to the list of available buffers.
656 * @param pBuffer the raw buffer to release
657 * @return >=0 if succeeded, neg. error code if failed
659 static int ReleaseRawBuffer(AliHLTRawBuffer* pBuffer);
662 * Deletes all the raw buffers.
663 * When the last Data Buffer object is destructed, all raw data buffers are
666 static int DeleteRawBuffers();
669 * Number of instances of AliHLTDataBuffer.
670 * The statice variable is incremented and decremented in the constructor/
671 * destructor. All internal data structures are cleaned up when the last
672 * instance is exiting.
674 static int fgNofInstances; // see above
675 /** global list of free raw buffers */
676 static vector<AliHLTRawBuffer*> fgFreeBuffers; // see above
677 /** global list of currently active raw buffers */
678 static vector<AliHLTRawBuffer*> fgActiveBuffers; // see above
679 /** determines the raw buffer size margin at buffer requests */
680 static AliHLTUInt32_t fgMargin; // see above
682 /** global instance to HLT logging class for static methods */
683 static AliHLTLogging fgLogging; // see above
685 /** size of the safety pattern */
686 static const Int_t fgkSafetyPatternSize; // see above
688 /** the safety pattern */
689 static const char fgkSafetyPattern[]; //!transient
691 static AliHLTUInt32_t fgEventCount; //!transient
693 //////////////////////////////////////////////////////////////////////////////
694 // internal helper functions
697 * Find the consumer descriptor for a certain component and data type in
698 * a list of consumers.<br>
699 * <b>Note:</b> There are three lists which contain the consumers in the
701 * @param pConsumer pointer to consumer component
702 * @param list list where to search for the consumer
704 AliHLTConsumerDescriptor* FindConsumer(const AliHLTComponent* pConsumer,
705 AliHLTConsumerDescriptorPList &list) const;
708 * Change the state of a consumer.
709 * The state of a consumer is determined by the list it is strored in, the
710 * method moves a consumer from the source to the target list.
711 * @param pDesc pointer to consumer descriptor
712 * @param srcList list where the consumer is currently to be found
713 * @param tgtList list where to move the consumer
715 int ChangeConsumerState(AliHLTConsumerDescriptor* pDesc,
716 AliHLTConsumerDescriptorPList &srcList,
717 AliHLTConsumerDescriptorPList &tgtList);
720 * Cleanup a consumer list.
721 * Release all allocated data structures. <b>Note:</b> Not the component itself!
723 int CleanupConsumerList();
725 ClassDef(AliHLTDataBuffer, 1)
728 #endif // ALIHLTDATABUFFER_H