casacore
Loading...
Searching...
No Matches
StArrayFile.h
Go to the documentation of this file.
1//# StArrayFile.h: Read/write array in external format for a storage manager
2//# Copyright (C) 1994,1995,1996,1997,1999,2001,2002
3//# Associated Universities, Inc. Washington DC, USA.
4//#
5//# This library is free software; you can redistribute it and/or modify it
6//# under the terms of the GNU Library General Public License as published by
7//# the Free Software Foundation; either version 2 of the License, or (at your
8//# option) any later version.
9//#
10//# This library is distributed in the hope that it will be useful, but WITHOUT
11//# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12//# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13//# License for more details.
14//#
15//# You should have received a copy of the GNU Library General Public License
16//# along with this library; if not, write to the Free Software Foundation,
17//# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18//#
19//# Correspondence concerning AIPS++ should be addressed as follows:
20//# Internet email: aips2-request@nrao.edu.
21//# Postal address: AIPS++ Project Office
22//# National Radio Astronomy Observatory
23//# 520 Edgemont Road
24//# Charlottesville, VA 22903-2475 USA
25//#
26//# $Id$
27
28#ifndef TABLES_STARRAYFILE_H
29#define TABLES_STARRAYFILE_H
30
31//# Includes
32#include <casacore/casa/aips.h>
33#include <casacore/casa/IO/RegularFileIO.h>
34#include <casacore/casa/IO/TypeIO.h>
35#include <casacore/casa/BasicSL/String.h>
36#include <casacore/casa/BasicSL/Complex.h>
37
38namespace casacore { //# NAMESPACE CASACORE - BEGIN
39
40//# Forward Declarations
41class MultiFileBase;
42class IPosition;
43
44
45// <summary>
46// Read/write array in external format for a storage manager
47// </summary>
48
49// <use visibility=local>
50
51// <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
52// </reviewed>
53
54// <prerequisite>
55//# Classes you should understand before using this one.
56// <li> ToLocal
57// <li> FromLocal
58// </prerequisite>
59
60// <etymology>
61// StManArrayFile is a class used by table storage managers
62// to store indirect arrays in a file.
63// </etymology>
64
65// <synopsis>
66// StManArrayFile is for use by the table storage manager, in particular
67// to read/write indirectly stored arrays.
68// Instead of holding the data in memory, they are written directly
69// into a file. It also allows to access a part of an array, which
70// is needed for the table system to access an array section.
71// It does not use a cache of its own, but it is relying on the
72// underlying system routines to cache and buffer adequately.
73//
74// This class could in principle also be used for other array purposes,
75// for example, to implement a paged array class for really huge arrays.
76//
77// An StManArrayFile object is connected to one file. It is possible
78// to hold multiple arrays in the file, each with its own shape.
79// An array is stored as its shape followed by the actual data
80// (all in little or big endian format). An array of strings is written as
81// an array of offsets pointing to the actual strings.
82// When a string gets a new value, the new value is written at the
83// end of the file and the file space with the old value is lost.
84//
85// Currently only the basic types are supported, but arbitrary types
86// could also be supported by writing/reading an element in the normal
87// way into the AipsIO buffer. It would only require that AipsIO
88// would contain a function to get its buffers and to restart them.
89// </synopsis>
90
91// <example>
92// <srcblock>
93// void writeArray (const Array<Bool>& array)
94// {
95// // Construct object and update file StArray.dat.
96// StManArrayFile arrayFile("StArray.dat, ByteIO::New);
97// // Reserve space for an array with the given shape and data type.
98// // This writes the shape at the end of the file and reserves
99// // space the hold the entire Bool array.
100// // It fills in the file offset where the shape is stored
101// // and returns the length of the shape in the file.
102// Int64 offset;
103// uInt shapeLength = arrayFile.putShape (array.shape(), offset, static_cast<Bool*>(0));
104// // Now put the actual array.
105// // This has to be put at the returned file offset plus the length
106// // of the shape in the file.
107// Bool deleteIt;
108// const Bool* dataPtr = array.getStorage (deleteIt);
109// arrayFile.put (offset+shapeLength, 0, array.nelements(), dataPtr);
110// array.freeStorage (dataPtr, deleteIt);
111// }
112// </srcblock>
113// </example>
114
115// <motivation>
116// The AipsIO class was not suitable for indirect table arrays,
117// because it uses memory to hold the data. Furthermore it is
118// not possible to access part of the data in AipsIO.
119// </motivation>
120
121// <todo asof="$DATE:$">
122// <li> implement long double
123// <li> support arbitrary types
124// <li> when rewriting a string value, use the current file
125// space if it fits
126// </todo>
127
128
130{
131public:
132
133 // Construct the object and attach it to the give file.
134 // The OpenOption determines how the file is opened
135 // (e.g. ByteIO::New for a new file).
136 // The buffersize is used to allocate a buffer of a proper size
137 // for the underlying filebuf object (see iostream package).
138 // A bufferSize 0 means using the default size (currently 65536).
140 uInt version=0, Bool bigEndian=True,
141 uInt bufferSize=0,
142 MultiFileBase* mfile=0);
143
144 // Close the possibly opened file.
146
147 // Flush and optionally fsync the data.
148 // It returns True when any data was written since the last flush.
149 Bool flush (Bool fsync);
150
151 // Reopen the file for read/write access.
152 void reopenRW();
153
154 // Resync the file (i.e. clear possible cache information).
155 void resync();
156
157 // Return the current file length (merely a debug tool).
159 { return leng_p; }
160
161 // Put the array shape and store its file offset into the offset argument.
162 // Reserve file space for the associated array.
163 // The length of the shape part in the file is returned.
164 // The file offset plus the shape length is the starting offset of the
165 // actual array data (which can be used by get and put).
166 // Space is reserved to store the reference count.
167 // <group>
168 uInt putShape (const IPosition& shape, Int64& fileOffset,
169 const Bool* dummy);
170 uInt putShape (const IPosition& shape, Int64& fileOffset,
171 const Char* dummy);
172 uInt putShape (const IPosition& shape, Int64& fileOffset,
173 const uChar* dummy);
174 uInt putShape (const IPosition& shape, Int64& fileOffset,
175 const Short* dummy);
176 uInt putShape (const IPosition& shape, Int64& fileOffset,
177 const uShort* dummy);
178 uInt putShape (const IPosition& shape, Int64& fileOffset,
179 const Int* dummy);
180 uInt putShape (const IPosition& shape, Int64& fileOffset,
181 const uInt* dummy);
182 uInt putShape (const IPosition& shape, Int64& fileOffset,
183 const Int64* dummy);
184 uInt putShape (const IPosition& shape, Int64& fileOffset,
185 const uInt64* dummy);
186 uInt putShape (const IPosition& shape, Int64& fileOffset,
187 const Float* dummy);
188 uInt putShape (const IPosition& shape, Int64& fileOffset,
189 const Double* dummy);
190 uInt putShape (const IPosition& shape, Int64& fileOffset,
191 const Complex* dummy);
192 uInt putShape (const IPosition& shape, Int64& fileOffset,
193 const DComplex* dummy);
194 uInt putShape (const IPosition& shape, Int64& fileOffset,
195 const String* dummy);
196 // </group>
197
198 // Get the reference count.
200
201 // Put the reference count.
202 // An exception is thrown if a value other than 1 is put for version 0.
203 void putRefCount (uInt refCount, Int64 offset);
204
205 // Put nr elements at the given file offset and array offset.
206 // The file offset of the first array element is the file offset
207 // of the shape plus the length of the shape in the file.
208 // The array offset is counted in number of elements. It can be
209 // used to put only a (contiguous) section of the array.
210 // <group>
211 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Bool*);
212 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Char*);
213 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uChar*);
214 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Short*);
215 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uShort*);
216 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Int*);
217 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uInt*);
218 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Int64*);
219 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uInt64*);
220 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Float*);
221 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Double*);
222//#// void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const long double*);
223 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Complex*);
224 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const DComplex*);
225 void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const String*);
226 // </group>
227
228 // Get the shape at the given file offset.
229 // It will reshape the IPosition vector when needed.
230 // It returns the length of the shape in the file.
232
233 // Get nr elements at the given file offset and array offset.
234 // The file offset of the first array element is the file offset
235 // of the shape plus the length of the shape in the file.
236 // The array offset is counted in number of elements. It can be
237 // used to get only a (contiguous) section of the array.
238 // <group>
239 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Bool*);
240 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Char*);
241 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uChar*);
242 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Short*);
243 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uShort*);
244 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Int*);
245 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uInt*);
246 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Int64*);
247 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uInt64*);
248 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Float*);
249 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Double*);
250//#// void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, long double*);
251 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Complex*);
252 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, DComplex*);
253 void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, String*);
254 // </group>
255
256 // Copy the array with <src>nr</src> elements from one file offset
257 // to another.
258 // <group>
259 void copyArrayBool (Int64 to, Int64 from, uInt64 nr);
260 void copyArrayChar (Int64 to, Int64 from, uInt64 nr);
261 void copyArrayuChar (Int64 to, Int64 from, uInt64 nr);
262 void copyArrayShort (Int64 to, Int64 from, uInt64 nr);
263 void copyArrayuShort (Int64 to, Int64 from, uInt64 nr);
264 void copyArrayInt (Int64 to, Int64 from, uInt64 nr);
265 void copyArrayuInt (Int64 to, Int64 from, uInt64 nr);
266 void copyArrayInt64 (Int64 to, Int64 from, uInt64 nr);
267 void copyArrayuInt64 (Int64 to, Int64 from, uInt64 nr);
268 void copyArrayFloat (Int64 to, Int64 from, uInt64 nr);
269 void copyArrayDouble (Int64 to, Int64 from, uInt64 nr);
270//#// void copyArrayLDouble (Int64 to, Int64 from, uInt64 nr);
271 void copyArrayComplex (Int64 to, Int64 from, uInt64 nr);
272 void copyArrayDComplex (Int64 to, Int64 from, uInt64 nr);
273 void copyArrayString (Int64 to, Int64 from, uInt64 nr);
274 // </group>
275
276private:
277 ByteIO* file_p; //# File object
278 TypeIO* iofil_p; //# IO object
279 Int64 leng_p; //# File length
280 uInt version_p; //# Version of StArrayFile file
281 Bool swput_p; //# True = put is possible
282 Bool hasPut_p; //# True = put since last flush
293
294 // Put a single value at the current file offset.
295 // It returns the length of the value in the file.
296 // <group>
297 uInt put (const Int&);
298 uInt put (const uInt&);
299 // </group>
300
301 // Put the array shape at the end of the file and reserve
302 // space for nr elements (each lenElem bytes long).
303 // It fills the file offset of the shape.
304 // It returns the length of the shape in the file.
305 uInt putRes (const IPosition& shape, Int64& fileOffset, float lenElem);
306
307 // Get a single value at the current file offset.
308 // It returns the length of the value in the file.
309 // <group>
310 uInt get (Int&);
311 uInt get (uInt&);
312 // </group>
313
314 // Copy data with the given length from one file offset to another.
315 void copyData (Int64 to, Int64 from, uInt64 length);
316
317 // Position the file on the given offset.
318 void setpos (Int64 offset);
319};
320
321
323{
324 file_p->reopenRW();
325}
327{
328 hasPut_p = True;
329 return iofil_p->write (1, &value);
330}
332{
333 hasPut_p = True;
334 return iofil_p->write (1, &value);
335}
337{
338 return iofil_p->read (1, &value);
339}
341{
342 return iofil_p->read (1, &value);
343}
344
345
346
347} //# NAMESPACE CASACORE - END
348
349#endif
virtual void reopenRW()
Reopen the underlying IO stream for read/write access.
OpenOption
Define the possible ByteIO open options.
Definition ByteIO.h:65
Abstract base class to combine multiple files in a single one.
void copyArrayInt64(Int64 to, Int64 from, uInt64 nr)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Char *)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uInt *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Char *dummy)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Bool *)
Put nr elements at the given file offset and array offset.
void copyArrayComplex(Int64 to, Int64 from, uInt64 nr)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, DComplex *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const String *dummy)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uChar *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Bool *dummy)
Put the array shape and store its file offset into the offset argument.
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Short *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Bool *)
Get nr elements at the given file offset and array offset.
void copyArrayChar(Int64 to, Int64 from, uInt64 nr)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Float *)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Int64 *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Complex *dummy)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const DComplex *dummy)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uShort *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Double *dummy)
void copyArrayFloat(Int64 to, Int64 from, uInt64 nr)
void resync()
Resync the file (i.e.
void copyData(Int64 to, Int64 from, uInt64 length)
Copy data with the given length from one file offset to another.
void copyArrayDouble(Int64 to, Int64 from, uInt64 nr)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Double *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Short *dummy)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Char *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Int *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Complex *)
void putRefCount(uInt refCount, Int64 offset)
Put the reference count.
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Complex *)
void copyArrayInt(Int64 to, Int64 from, uInt64 nr)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const uChar *dummy)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Int *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uInt *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Float *dummy)
void setpos(Int64 offset)
Position the file on the given offset.
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const String *)
void copyArrayBool(Int64 to, Int64 from, uInt64 nr)
Copy the array with nr elements from one file offset to another.
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Short *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Int *dummy)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uShort *)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uChar *)
Bool flush(Bool fsync)
Flush and optionally fsync the data.
uInt putShape(const IPosition &shape, Int64 &fileOffset, const uInt *dummy)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Int64 *dummy)
uInt putRes(const IPosition &shape, Int64 &fileOffset, float lenElem)
Put the array shape at the end of the file and reserve space for nr elements (each lenElem bytes long...
uInt putShape(const IPosition &shape, Int64 &fileOffset, const uShort *dummy)
void copyArrayuInt(Int64 to, Int64 from, uInt64 nr)
StManArrayFile(const String &name, ByteIO::OpenOption, uInt version=0, Bool bigEndian=True, uInt bufferSize=0, MultiFileBase *mfile=0)
Construct the object and attach it to the give file.
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Double *)
~StManArrayFile()
Close the possibly opened file.
uInt getShape(Int64 fileOffset, IPosition &shape)
Get the shape at the given file offset.
uInt getRefCount(Int64 offset)
Get the reference count.
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uInt64 *)
void reopenRW()
Reopen the file for read/write access.
Int64 length()
Return the current file length (merely a debug tool).
void copyArrayuInt64(Int64 to, Int64 from, uInt64 nr)
void copyArrayuChar(Int64 to, Int64 from, uInt64 nr)
void copyArrayShort(Int64 to, Int64 from, uInt64 nr)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const uInt64 *dummy)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uInt64 *)
void copyArrayDComplex(Int64 to, Int64 from, uInt64 nr)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Float *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Int64 *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, String *)
void copyArrayString(Int64 to, Int64 from, uInt64 nr)
void copyArrayuShort(Int64 to, Int64 from, uInt64 nr)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const DComplex *)
String: the storage and methods of handling collections of characters.
Definition String.h:225
virtual size_t write(size_t nvalues, const Bool *value)
Convert the values and write them to the ByteIO object.
virtual size_t read(size_t nvalues, Bool *value)
Read the values from the ByteIO object and convert them.
this file contains all the compiler specific defines
Definition mainpage.dox:28
unsigned char uChar
Definition aipstype.h:47
short Short
Definition aipstype.h:48
unsigned int uInt
Definition aipstype.h:51
unsigned short uShort
Definition aipstype.h:49
TableExprNode shape(const TableExprNode &array)
Function operating on any scalar or array resulting in a Double array containing the shape.
Definition ExprNode.h:1987
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition aipsxtype.h:38
float Float
Definition aipstype.h:54
int Int
Definition aipstype.h:50
bool Bool
Define the standard types used by Casacore.
Definition aipstype.h:42
LatticeExprNode value(const LatticeExprNode &expr)
This function returns the value of the expression without a mask.
const Bool True
Definition aipstype.h:43
double Double
Definition aipstype.h:55
char Char
Definition aipstype.h:46
unsigned long long uInt64
Definition aipsxtype.h:39