xrootd
XrdPfcInfo.hh
Go to the documentation of this file.
1 #ifndef __XRDPFC_INFO_HH__
2 #define __XRDPFC_INFO_HH__
3 //----------------------------------------------------------------------------------
4 // Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
5 // Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman
6 //----------------------------------------------------------------------------------
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //----------------------------------------------------------------------------------
20 
21 #include <stdio.h>
22 #include <time.h>
23 #include <assert.h>
24 #include <vector>
25 
26 #include "XrdSys/XrdSysPthread.hh"
27 #include "XrdCl/XrdClConstants.hh"
28 #include "XrdCl/XrdClDefaultEnv.hh"
29 
30 class XrdOssDF;
31 class XrdCksCalc;
32 class XrdSysTrace;
33 
34 
35 namespace XrdCl
36 {
37 class Log;
38 }
39 
40 namespace XrdPfc
41 {
42 class Stats;
43 
44 //----------------------------------------------------------------------------
46 //----------------------------------------------------------------------------
47 
48 class Info
49 {
50 public:
52  struct AStat
53  {
54  time_t AttachTime;
55  time_t DetachTime;
56  int NumIos;
57  int Duration;
58  int NumMerged;
59  long long BytesHit;
60  long long BytesMissed;
61  long long BytesBypassed;
62 
63  AStat() :
64  AttachTime(0), DetachTime(0), NumIos(0), Duration(0), NumMerged(0),
66  {}
67 
68  void MergeWith(const AStat &a);
69  };
70 
71  struct Store
72  {
73  int m_version;
74  long long m_buffer_size;
75  long long m_file_size;
76  unsigned char *m_buff_synced;
77  char m_cksum[16];
78  time_t m_creationTime;
79  size_t m_accessCnt;
80  std::vector<AStat> m_astats;
81 
83  };
84 
85 
86  //------------------------------------------------------------------------
88  //------------------------------------------------------------------------
89  Info(XrdSysTrace* trace, bool prefetchBuffer = false);
90 
91  //------------------------------------------------------------------------
93  //------------------------------------------------------------------------
94  ~Info();
95 
96  //---------------------------------------------------------------------
98  //---------------------------------------------------------------------
99  void SetBitWritten(int i);
100 
101  //---------------------------------------------------------------------
103  //---------------------------------------------------------------------
104  bool TestBitWritten(int i) const;
105 
106  //---------------------------------------------------------------------
108  //---------------------------------------------------------------------
109  bool TestBitPrefetch(int i) const;
110 
111  //---------------------------------------------------------------------
113  //---------------------------------------------------------------------
114  void SetBitPrefetch(int i);
115 
116  //---------------------------------------------------------------------
118  //---------------------------------------------------------------------
119  void SetBitSynced(int i);
120 
121  //---------------------------------------------------------------------
123  //---------------------------------------------------------------------
125 
126  void SetBufferSize(long long);
127 
128  void SetFileSize(long long);
129 
130  //---------------------------------------------------------------------
134  //---------------------------------------------------------------------
135  void ResizeBits(int n);
136 
137  //---------------------------------------------------------------------
144  //---------------------------------------------------------------------
145  bool Read(XrdOssDF* fp, const std::string &fname = "<unknown>");
146 
147  //---------------------------------------------------------------------
150  //---------------------------------------------------------------------
151  bool Write(XrdOssDF* fp, const std::string &fname = "<unknown>");
152 
153  //---------------------------------------------------------------------
155  //---------------------------------------------------------------------
157 
158  //---------------------------------------------------------------------
160  //---------------------------------------------------------------------
162 
163  //---------------------------------------------------------------------
165  //---------------------------------------------------------------------
167 
168  //---------------------------------------------------------------------
170  //---------------------------------------------------------------------
172 
173  //---------------------------------------------------------------------
175  //---------------------------------------------------------------------
176  void WriteIOStat(Stats& s);
177 
178  //---------------------------------------------------------------------
180  //---------------------------------------------------------------------
182 
183  //---------------------------------------------------------------------
185  //---------------------------------------------------------------------
186  void WriteIOStatSingle(long long bytes_disk);
187 
188  //---------------------------------------------------------------------
190  //---------------------------------------------------------------------
191  void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc);
192 
193  //---------------------------------------------------------------------
195  //---------------------------------------------------------------------
196  bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const;
197 
198  //---------------------------------------------------------------------
200  //---------------------------------------------------------------------
201  int GetSizeInBytes() const;
202 
203  //---------------------------------------------------------------------
205  //---------------------------------------------------------------------
206  int GetSizeInBits() const;
207 
208  //---------------------------------------------------------------------
210  //---------------------------------------------------------------------
211  long long GetFileSize() const;
212 
213  //---------------------------------------------------------------------
215  //---------------------------------------------------------------------
216  bool GetLatestDetachTime(time_t& t) const;
217 
218  //---------------------------------------------------------------------
220  //---------------------------------------------------------------------
221  const AStat* GetLastAccessStats() const;
222 
223  //---------------------------------------------------------------------
225  //---------------------------------------------------------------------
226  long long GetBufferSize() const;
227 
228  //---------------------------------------------------------------------
230  //---------------------------------------------------------------------
231  bool IsComplete() const;
232 
233  //---------------------------------------------------------------------
235  //---------------------------------------------------------------------
236  int GetNDownloadedBlocks() const;
237 
238  //---------------------------------------------------------------------
240  //---------------------------------------------------------------------
241  long long GetNDownloadedBytes() const;
242 
243  //---------------------------------------------------------------------
245  //---------------------------------------------------------------------
246  int GetLastDownloadedBlock() const;
247 
248  //---------------------------------------------------------------------
250  //---------------------------------------------------------------------
251  long long GetExpectedDataFileSize() const;
252 
253  //---------------------------------------------------------------------
255  //---------------------------------------------------------------------
257 
258  //---------------------------------------------------------------------
260  //---------------------------------------------------------------------
261  size_t GetAccessCnt() const { return m_store.m_accessCnt; }
262 
263  //---------------------------------------------------------------------
265  //---------------------------------------------------------------------
266  int GetVersion() { return m_store.m_version; }
267 
268  //---------------------------------------------------------------------
270  //---------------------------------------------------------------------
271  const Store& RefStoredData() const { return m_store; }
272 
273  //---------------------------------------------------------------------
275  //---------------------------------------------------------------------
276  void GetCksum( unsigned char* buff, char* digest);
277 
278  static const char* m_traceID; // has to be m_ (convention in TRACE macros)
279  static const char* s_infoExtension;
280  static const int s_defaultVersion;
281  static size_t s_maxNumAccess; // can be set from configuration
282 
283  XrdSysTrace* GetTrace() const {return m_trace; }
284 
285 protected:
287 
290  unsigned char *m_buff_written;
291  unsigned char *m_buff_prefetch;
292 
294  bool m_complete;
295 
296 private:
297  inline unsigned char cfiBIT(int n) const { return 1 << n; }
298 
299  // Reading functions for older cinfo file formats
300  bool ReadV1(XrdOssDF* fp, const std::string &fname);
301  bool ReadV2(XrdOssDF* fp, const std::string &fname);
302 
304 };
305 
306 //------------------------------------------------------------------------------
307 
308 inline bool Info::TestBitWritten(int i) const
309 {
310  const int cn = i/8;
311  assert(cn < GetSizeInBytes());
312 
313  const int off = i - cn*8;
314  return (m_buff_written[cn] & cfiBIT(off)) != 0;
315 }
316 
317 inline void Info::SetBitWritten(int i)
318 {
319  const int cn = i/8;
320  assert(cn < GetSizeInBytes());
321 
322  const int off = i - cn*8;
323  m_buff_written[cn] |= cfiBIT(off);
324 }
325 
326 inline void Info::SetBitPrefetch(int i)
327 {
328  if (!m_buff_prefetch) return;
329 
330  const int cn = i/8;
331  assert(cn < GetSizeInBytes());
332 
333  const int off = i - cn*8;
334  m_buff_prefetch[cn] |= cfiBIT(off);
335 }
336 
337 inline bool Info::TestBitPrefetch(int i) const
338 {
339  if (!m_buff_prefetch) return false;
340 
341  const int cn = i/8;
342  assert(cn < GetSizeInBytes());
343 
344  const int off = i - cn*8;
345  return (m_buff_prefetch[cn] & cfiBIT(off)) != 0;
346 }
347 
348 inline void Info::SetBitSynced(int i)
349 {
350  const int cn = i/8;
351  assert(cn < GetSizeInBytes());
352 
353  const int off = i - cn*8;
354  m_store.m_buff_synced[cn] |= cfiBIT(off);
355 }
356 
357 //------------------------------------------------------------------------------
358 
359 inline int Info::GetNDownloadedBlocks() const
360 {
361  int cntd = 0;
362  for (int i = 0; i < m_sizeInBits; ++i)
363  if (TestBitWritten(i)) cntd++;
364 
365  return cntd;
366 }
367 
368 inline long long Info::GetNDownloadedBytes() const
369 {
371 }
372 
374 {
375  for (int i = m_sizeInBits - 1; i >= 0; --i)
376  if (TestBitWritten(i)) return i;
377 
378  return -1;
379 }
380 
381 inline long long Info::GetExpectedDataFileSize() const
382 {
383  int last_block = GetLastDownloadedBlock();
384  if (last_block == m_sizeInBits - 1)
385  return m_store.m_file_size;
386  else
387  return (last_block + 1) * m_store.m_buffer_size;
388 }
389 
390 inline int Info::GetSizeInBytes() const
391 {
392  if (m_sizeInBits)
393  return ((m_sizeInBits - 1)/8 + 1);
394  else
395  return 0;
396 }
397 
398 inline int Info::GetSizeInBits() const
399 {
400  return m_sizeInBits;
401 }
402 
403 inline long long Info::GetFileSize() const
404 {
405  return m_store.m_file_size;
406 }
407 
408 inline bool Info::IsComplete() const
409 {
410  return m_complete;
411 }
412 
413 inline bool Info::IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
414 {
415  // TODO rewrite to use full byte comparisons outside of edges ?
416  // Also, it is always called with fisrtsdx = 0, lastIdx = m_sizeInBits.
417  for (int i = firstIdx; i < lastIdx; ++i)
418  if (! TestBitWritten(i)) return true;
419 
420  return false;
421 }
422 
424 {
426 }
427 
428 inline long long Info::GetBufferSize() const
429 {
430  return m_store.m_buffer_size;
431 }
432 
433 }
434 #endif
XrdClConstants.hh
XrdPfc::Info::Store::m_cksum
char m_cksum[16]
cksum of downloaded information
Definition: XrdPfcInfo.hh:77
XrdPfc::Info::m_trace
XrdSysTrace * m_trace
Definition: XrdPfcInfo.hh:286
XrdPfc::Info::s_infoExtension
static const char * s_infoExtension
Definition: XrdPfcInfo.hh:279
XrdPfc::Info::AStat::Duration
int Duration
total duration of all IOs attached
Definition: XrdPfcInfo.hh:57
XrdPfc::Info::AStat::NumIos
int NumIos
number of IO objects attached during this access
Definition: XrdPfcInfo.hh:56
XrdPfc::Info::SetBitSynced
void SetBitSynced(int i)
Mark block as synced to disk.
Definition: XrdPfcInfo.hh:348
XrdPfc::Info::Info
Info(XrdSysTrace *trace, bool prefetchBuffer=false)
Constructor.
XrdPfc::Info::GetLastAccessStats
const AStat * GetLastAccessStats() const
Get latest access stats.
XrdSysPthread.hh
XrdPfc::Info::Store::m_astats
std::vector< AStat > m_astats
access records
Definition: XrdPfcInfo.hh:80
XrdPfc::Info::SetBitPrefetch
void SetBitPrefetch(int i)
Mark block as obtained through prefetch.
Definition: XrdPfcInfo.hh:326
XrdPfc::Info::GetFileSize
long long GetFileSize() const
Get file size.
Definition: XrdPfcInfo.hh:403
XrdPfc::Info::SetBitWritten
void SetBitWritten(int i)
Mark block as written to disk.
Definition: XrdPfcInfo.hh:317
XrdPfc::Info::UpdateDownloadCompleteStatus
void UpdateDownloadCompleteStatus()
Update complete status.
Definition: XrdPfcInfo.hh:423
XrdPfc::Info::GetLatestDetachTime
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
XrdPfc::Info::GetSizeInBytes
int GetSizeInBytes() const
Get size of download-state bit-vector in bytes.
Definition: XrdPfcInfo.hh:390
XrdPfc::Info::Store::m_creationTime
time_t m_creationTime
time the info file was created
Definition: XrdPfcInfo.hh:78
XrdPfc::Info::WriteIOStatDetach
void WriteIOStatDetach(Stats &s)
Write close time together with bytes missed, hits, and disk.
XrdPfc::Info::Store::m_version
int m_version
info version
Definition: XrdPfcInfo.hh:73
XrdPfc::Info::m_traceID
static const char * m_traceID
Definition: XrdPfcInfo.hh:278
XrdPfc::Info::Store::m_accessCnt
size_t m_accessCnt
total access count for the file
Definition: XrdPfcInfo.hh:79
XrdPfc::Info::IsAnythingEmptyInRng
bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
Check download status in given block range.
Definition: XrdPfcInfo.hh:413
XrdPfc::Info::TestBitWritten
bool TestBitWritten(int i) const
Test if block at the given index is written to disk.
Definition: XrdPfcInfo.hh:308
XrdOssDF
Definition: XrdOss.hh:62
XrdPfc::Info::GetSizeInBits
int GetSizeInBits() const
Get number of blocks represented in download-state bit-vector.
Definition: XrdPfcInfo.hh:398
XrdPfc::Info::AStat::DetachTime
time_t DetachTime
close time
Definition: XrdPfcInfo.hh:55
XrdPfc::Info::WriteIOStat
void WriteIOStat(Stats &s)
Write bytes missed, hits, and disk.
XrdPfc::Info::IsComplete
bool IsComplete() const
Get complete status.
Definition: XrdPfcInfo.hh:408
XrdPfc::Info::Store
Definition: XrdPfcInfo.hh:72
XrdPfc::Info::ResizeBits
void ResizeBits(int n)
Reserve buffer for file_size / buffer_size bytes.
XrdPfc::Info::m_sizeInBits
int m_sizeInBits
cached
Definition: XrdPfcInfo.hh:293
XrdPfc::Info::RefStoredData
const Store & RefStoredData() const
Get stored data.
Definition: XrdPfcInfo.hh:271
XrdPfc::Info::GetCksum
void GetCksum(unsigned char *buff, char *digest)
Get md5 cksum.
XrdPfc::Info::ReadV1
bool ReadV1(XrdOssDF *fp, const std::string &fname)
XrdPfc::Info::TestBitPrefetch
bool TestBitPrefetch(int i) const
Test if block at the given index has been prefetched.
Definition: XrdPfcInfo.hh:337
XrdPfc::Info::AStat::AStat
AStat()
Definition: XrdPfcInfo.hh:63
XrdPfc::Info::AStat::AttachTime
time_t AttachTime
open time
Definition: XrdPfcInfo.hh:54
XrdPfc::Info::WriteIOStatAttach
void WriteIOStatAttach()
Write open time in the last entry of access statistics.
XrdPfc::Info::SetFileSize
void SetFileSize(long long)
XrdPfc::Info::ReadV2
bool ReadV2(XrdOssDF *fp, const std::string &fname)
XrdPfc::Info::~Info
~Info()
Destructor.
XrdPfc::Info::AStat::MergeWith
void MergeWith(const AStat &a)
XrdPfc::Info::WriteIOStatSingle
void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc)
Write open/close with given time and bytes read from disk.
XrdPfc::Info::Store::m_file_size
long long m_file_size
number of file blocks
Definition: XrdPfcInfo.hh:75
XrdPfc::Info::DisableDownloadStatus
void DisableDownloadStatus()
Disable allocating, writing, and reading of download status.
XrdPfc::Info::Store::m_buffer_size
long long m_buffer_size
prefetch buffer size
Definition: XrdPfcInfo.hh:74
XrdPfc::Info::Read
bool Read(XrdOssDF *fp, const std::string &fname="<unknown>")
Rea load content from cinfo file into this object.
XrdPfc::Info::GetExpectedDataFileSize
long long GetExpectedDataFileSize() const
Get expected data file size.
Definition: XrdPfcInfo.hh:381
XrdPfc::Info::GetVersion
int GetVersion()
Get version.
Definition: XrdPfcInfo.hh:266
XrdPfc::Info::AStat
Access statistics.
Definition: XrdPfcInfo.hh:53
XrdPfc::Info::Store::Store
Store()
Definition: XrdPfcInfo.hh:82
XrdPfc::Info::Write
bool Write(XrdOssDF *fp, const std::string &fname="<unknown>")
XrdPfc::Info
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:49
XrdPfc::Info::m_complete
bool m_complete
cached
Definition: XrdPfcInfo.hh:294
XrdCl
Definition: XrdClAnyObject.hh:26
XrdPfc::Info::GetTrace
XrdSysTrace * GetTrace() const
Definition: XrdPfcInfo.hh:283
XrdClDefaultEnv.hh
XrdPfc::Info::ResetAllAccessStats
void ResetAllAccessStats()
Reset IO Stats.
XrdPfc::Info::m_hasPrefetchBuffer
bool m_hasPrefetchBuffer
constains current prefetch score
Definition: XrdPfcInfo.hh:289
XrdPfc::Info::m_cksCalc
XrdCksCalc * m_cksCalc
Definition: XrdPfcInfo.hh:303
XrdPfc::Info::GetNDownloadedBytes
long long GetNDownloadedBytes() const
Get number of downloaded bytes.
Definition: XrdPfcInfo.hh:368
XrdPfc::Info::Store::m_buff_synced
unsigned char * m_buff_synced
disk written state vector
Definition: XrdPfcInfo.hh:76
XrdPfc::Info::AStat::NumMerged
int NumMerged
number of times the record has been merged
Definition: XrdPfcInfo.hh:58
XrdPfc::Info::m_buff_written
unsigned char * m_buff_written
download state vector
Definition: XrdPfcInfo.hh:290
XrdPfc::Info::m_store
Store m_store
Definition: XrdPfcInfo.hh:288
XrdPfc::Info::s_defaultVersion
static const int s_defaultVersion
Definition: XrdPfcInfo.hh:280
XrdPfc::Stats
Statistics of cache utilisation by a File object.
Definition: XrdPfcStats.hh:31
XrdPfc::Info::CompactifyAccessRecords
void CompactifyAccessRecords()
Compactify access records to the configured maximum.
XrdPfc::Info::WriteIOStatSingle
void WriteIOStatSingle(long long bytes_disk)
Write single open/close time for given bytes read from disk.
XrdPfc::Info::AStat::BytesMissed
long long BytesMissed
read from remote and cached
Definition: XrdPfcInfo.hh:60
XrdPfc
Definition: XrdPfc.hh:46
XrdPfc::Info::AStat::BytesHit
long long BytesHit
read from cache
Definition: XrdPfcInfo.hh:59
XrdPfc::Info::GetBufferSize
long long GetBufferSize() const
Get prefetch buffer size.
Definition: XrdPfcInfo.hh:428
XrdSysTrace
Definition: XrdSysTrace.hh:49
XrdPfc::Info::m_buff_prefetch
unsigned char * m_buff_prefetch
prefetch statistics
Definition: XrdPfcInfo.hh:291
XrdPfc::Info::GetNDownloadedBlocks
int GetNDownloadedBlocks() const
Get number of downloaded blocks.
Definition: XrdPfcInfo.hh:359
XrdPfc::Info::SetBufferSize
void SetBufferSize(long long)
XrdCksCalc
Definition: XrdCksCalc.hh:40
XrdPfc::Info::AStat::BytesBypassed
long long BytesBypassed
read from remote and dropped
Definition: XrdPfcInfo.hh:61
XrdPfc::Info::s_maxNumAccess
static size_t s_maxNumAccess
Definition: XrdPfcInfo.hh:281
XrdPfc::Info::GetLastDownloadedBlock
int GetLastDownloadedBlock() const
Get number of the last downloaded block.
Definition: XrdPfcInfo.hh:373
XrdPfc::Info::SetAllBitsSynced
void SetAllBitsSynced()
Mark all blocks as synced to disk.
XrdPfc::Info::cfiBIT
unsigned char cfiBIT(int n) const
Definition: XrdPfcInfo.hh:297
XrdPfc::Info::GetAccessCnt
size_t GetAccessCnt() const
Get number of accesses.
Definition: XrdPfcInfo.hh:261