GRASS GIS 8 Programmer's Manual 8.4.1(2025)-45ca3179ab
Loading...
Searching...
No Matches
compress.c
Go to the documentation of this file.
1/*
2 ****************************************************************************
3 * -- GRASS Development Team --
4 *
5 * MODULE: GRASS gis library
6 * FILENAME: compress.c
7 * AUTHOR(S): Markus Metz
8 * PURPOSE: To provide an interface for compressing and
9 * decompressing data using various methods. Its primary
10 * use is in the storage and reading of GRASS rasters.
11 *
12 * DATE CREATED: Dec 17 2015
13 * COPYRIGHT: (C) 2015 by the GRASS Development Team
14 *
15 * This program is free software under the GNU General Public
16 * License (version 2 or greater). Read the file COPYING that
17 * comes with GRASS for details.
18 *
19 *****************************************************************************/
20
21/********************************************************************
22 * Compression methods: *
23 * 1 : RLE (generic Run-Length Encoding of single bytes) *
24 * 2 : ZLIB's DEFLATE (good speed and compression) *
25 * 3 : LZ4 (fastest, low compression) *
26 * 4 : BZIP2 (slowest, high compression) *
27 * 5 : ZSTD (faster than ZLIB, higher compression than ZLIB) *
28 * *
29 * int *
30 * G_read_compressed (fd, rbytes, dst, nbytes, compression_type) *
31 * int fd, rbytes, nbytes; *
32 * unsigned char *dst; *
33 * ---------------------------------------------------------------- *
34 * This is the basic function for reading a compressed chunk of a *
35 * data file. The file descriptor should be in the proper location *
36 * and the 'dst' array should have enough space for the data. *
37 * 'nbytes' is the size of 'dst'. The 'rbytes' parameter is the *
38 * number of bytes to read (knowable from the offsets index). For *
39 * best results, 'nbytes' should be the exact amount of space *
40 * needed for the expansion. Too large a value of nbytes may cause *
41 * more data to be expanded than is desired. *
42 * Returns: The number of bytes decompressed into dst, or an error. *
43 * *
44 * Errors include: *
45 * -1 -- Error Reading or Decompressing data. *
46 * -2 -- Not enough space in dst. You must make dst larger *
47 * and then call the function again (remembering to *
48 * reset the file descriptor to it's proper location. *
49 * *
50 * ================================================================ *
51 * int *
52 * G_write_compressed (fd, src, nbytes, compression_type) *
53 * int fd, nbytes; *
54 * unsigned char *src; *
55 * ---------------------------------------------------------------- *
56 * This is the basic function for writing and compressing a data *
57 * chunk to a file. The file descriptor should be in the correct *
58 * location prior to this call. The function will compress 'nbytes' *
59 * of 'src' and write it to the file 'fd'. Returns the number of *
60 * bytes written or an error code: *
61 * *
62 * Errors include: *
63 * -1 -- Compression Failed. *
64 * -2 -- Unable to write to file. *
65 * *
66 * ================================================================ *
67 * int *
68 * G_write_uncompressed (fd, src, nbytes) *
69 * int fd, nbytes; *
70 * unsigned char *src; *
71 * ---------------------------------------------------------------- *
72 * Works similar to G_write_compressed() except no attempt at *
73 * compression is made. This is quicker, but may result in larger *
74 * files. *
75 * Returns the number of bytes written, or -1 for an error. It will *
76 * return an error if it fails to write nbytes. Otherwise, the *
77 * return value will always be nbytes + 1 (for compression flag). *
78 * *
79 ********************************************************************
80 */
81
82#include <grass/config.h>
83
84#include <stdio.h>
85#include <stdlib.h>
86#include <string.h>
87#include <errno.h>
88#include <unistd.h>
89#include <grass/gis.h>
90#include <grass/glocale.h>
91
92#include "compress.h"
93
94#define G_COMPRESSED_NO (unsigned char)'0'
95#define G_COMPRESSED_YES (unsigned char)'1'
96
97/* get compressor number
98 * return -1 on error
99 * return number >= 0 for known processor */
101{
102 int i;
103
104 if (!name)
105 return -1;
106
107 for (i = 0; compressor[i].name; i++) {
108 if (G_strcasecmp(name, compressor[i].name) == 0)
109 return i;
110 }
111
112 return -1;
113}
114
115/* get compressor name
116 * return NULL on error
117 * return string (name) of known processor */
118char *G_compressor_name(int number)
119{
120 if (number < 0 || number >= n_compressors)
121 return NULL;
122
123 return compressor[number].name;
124}
125
127{
128#ifdef HAVE_ZSTD_H
129 /* ZSTD */
130 return 5;
131#endif
132 /* ZLIB */
133 return 2;
134}
135
136/* check compressor number
137 * return -1 on error
138 * return 0 known but not available
139 * return 1 known and available */
140int G_check_compressor(int number)
141{
142 if (number < 0 || number >= n_compressors) {
143 G_warning(_("Request for unsupported compressor"));
144 return -1;
145 }
146
147 return compressor[number].available;
148}
149
150int G_no_compress_bound(int src_sz)
151{
152 return src_sz;
153}
154
155int G_no_compress(unsigned char *src, int src_sz, unsigned char *dst,
156 int dst_sz)
157{
158 /* Catch errors early */
159 if (src == NULL || dst == NULL)
160 return -1;
161
162 /* Don't do anything if src is empty */
163 if (src_sz <= 0)
164 return 0;
165
166 /* dst too small */
167 if (dst_sz < src_sz)
168 return -2;
169
170 /* Copy the data from src to dst */
171 memcpy(dst, src, src_sz);
172
173 return src_sz;
174}
175
176int G_no_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz)
177{
178 /* Catch errors early */
179 if (src == NULL || dst == NULL)
180 return -1;
181
182 /* Don't do anything if src is empty */
183 if (src_sz <= 0)
184 return 0;
185
186 /* dst too small */
187 if (dst_sz < src_sz)
188 return -2;
189
190 /* Copy the data from src to dst */
191 memcpy(dst, src, src_sz);
192
193 return src_sz;
194}
195
196/* G_*_compress_bound() returns an upper bound on the compressed size
197 * which can be larger than the input size
198 * some compressors are a bit faster if the size of the destination
199 * is at least the upper bound (no need to test for buffer overflow)
200 * read comments on the specific compressor interfaces
201 */
202int G_compress_bound(int src_sz, int number)
203{
204 if (number < 0 || number >= n_compressors) {
205 G_fatal_error(_("Request for unsupported compressor"));
206 return -1;
207 }
208
209 return compressor[number].bound(src_sz);
210}
211
212/* G_*_compress() returns
213 * > 0: number of bytes in dst
214 * 0: nothing done
215 * -1: error
216 * -2: dst too small
217 */
218int G_compress(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz,
219 int number)
220{
221 if (number < 0 || number >= n_compressors) {
222 G_fatal_error(_("Request for unsupported compressor"));
223 return -1;
224 }
225
226 return compressor[number].compress(src, src_sz, dst, dst_sz);
227}
228
229/* G_*_expand() returns
230 * > 0: number of bytes in dst
231 * -1: error
232 */
233int G_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz,
234 int number)
235{
236 if (number < 0 || number >= n_compressors) {
237 G_fatal_error(_("Request for unsupported compressor"));
238 return -1;
239 }
240
241 return compressor[number].expand(src, src_sz, dst, dst_sz);
242}
243
244int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes,
245 int number)
246{
247 int bsize, nread, err;
248 unsigned char *b;
249
250 if (dst == NULL || nbytes <= 0) {
251 if (dst == NULL)
252 G_warning(_("No destination buffer allocated"));
253 if (nbytes <= 0)
254 G_warning(_("Invalid destination buffer size %d"), nbytes);
255 return -2;
256 }
257
258 if (rbytes <= 0) {
259 G_warning(_("Invalid read size %d"), nbytes);
260 return -2;
261 }
262
263 bsize = rbytes;
264
265 /* Our temporary input buffer for read */
266 if (NULL == (b = (unsigned char *)G_calloc(bsize, sizeof(unsigned char))))
267 return -1;
268
269 /* Read from the file until we get our bsize or an error */
270 nread = 0;
271 do {
272 err = read(fd, b + nread, bsize - nread);
273 if (err >= 0)
274 nread += err;
275 } while (err > 0 && nread < bsize);
276
277 if (err <= 0) {
278 if (err == 0)
279 G_warning(_("Unable to read %d bytes: end of file"), rbytes);
280 else
281 G_warning(_("Unable to read %d bytes: %s"), rbytes,
282 strerror(errno));
283 return -1;
284 }
285
286 /* If the bsize if less than rbytes and we didn't get an error.. */
287 if (nread < rbytes) {
288 G_free(b);
289 G_warning("Unable to read %d bytes, got %d bytes", rbytes, nread);
290 return -1;
291 }
292
293 /* Test if row is compressed */
294 if (b[0] == G_COMPRESSED_NO) {
295 /* Then just copy it to dst */
296 for (err = 0; err < nread - 1 && err < nbytes; err++)
297 dst[err] = b[err + 1];
298
299 G_free(b);
300 return (nread - 1);
301 }
302 else if (b[0] != G_COMPRESSED_YES) {
303 /* We're not at the start of a row */
304 G_free(b);
305 G_warning("Read error: We're not at the start of a row");
306 return -1;
307 }
308 /* Okay it's a compressed row */
309
310 /* Just call G_expand() with the buffer we read,
311 * Account for first byte being a flag
312 */
313 err = G_expand(b + 1, bsize - 1, dst, nbytes, number);
314
315 /* We're done with b */
316 G_free(b);
317
318 /* Return whatever G_expand() returned */
319 return err;
320
321} /* G_read_compressed() */
322
323int G_write_compressed(int fd, unsigned char *src, int nbytes, int number)
324{
325 int dst_sz, nwritten, err;
326 unsigned char *dst, compressed;
327
328 /* Catch errors */
329 if (src == NULL || nbytes < 0) {
330 if (src == NULL)
331 G_warning(_("No source buffer"));
332 if (nbytes <= 0)
333 G_warning(_("Invalid source buffer size %d"), nbytes);
334 return -1;
335 }
336
337 /* get upper bound of compressed size */
338 dst_sz = G_compress_bound(nbytes, number);
339 if (NULL ==
340 (dst = (unsigned char *)G_calloc(dst_sz, sizeof(unsigned char))))
341 return -1;
342
343 /* Now just call G_compress() */
344 err = G_compress(src, nbytes, dst, dst_sz, number);
345
346 /* If compression succeeded write compressed row,
347 * otherwise write uncompressed row. Compression will fail
348 * if dst is too small (i.e. compressed data is larger)
349 */
350 if (err > 0 && err < nbytes) {
351 dst_sz = err;
352 /* Write the compression flag */
353 compressed = G_COMPRESSED_YES;
354 if (write(fd, &compressed, 1) != 1) {
355 G_free(dst);
356 G_warning(_("Unable to write compression flag"));
357 return -1;
358 }
359 nwritten = 0;
360 do {
361 err = write(fd, dst + nwritten, dst_sz - nwritten);
362 if (err >= 0)
363 nwritten += err;
364 } while (err > 0 && nwritten < dst_sz);
365 if (err <= 0) {
366 if (err == 0)
367 G_warning(_("Unable to write %d bytes: nothing written"),
368 dst_sz);
369 else
370 G_warning(_("Unable to write %d bytes: %s"), dst_sz,
371 strerror(errno));
372 }
373 /* Account for extra byte */
374 nwritten++;
375 }
376 else {
377 /* Write compression flag */
378 compressed = G_COMPRESSED_NO;
379 if (write(fd, &compressed, 1) != 1) {
380 G_free(dst);
381 G_warning(_("Unable to write compression flag"));
382 return -1;
383 }
384 nwritten = 0;
385 do {
386 err = write(fd, src + nwritten, nbytes - nwritten);
387 if (err >= 0)
388 nwritten += err;
389 } while (err > 0 && nwritten < nbytes);
390 if (err <= 0) {
391 if (err == 0)
392 G_warning(_("Unable to write %d bytes: nothing written"),
393 nbytes);
394 else
395 G_warning(_("Unable to write %d bytes: %s"), nbytes,
396 strerror(errno));
397 }
398 /* Account for extra byte */
399 nwritten++;
400 } /* if (err > 0) */
401
402 /* Done with the dst buffer */
403 G_free(dst);
404
405 /* If we didn't write all the data return an error */
406 if (err < 0)
407 return -2;
408
409 return nwritten;
410} /* G_write_compressed() */
411
412int G_write_uncompressed(int fd, const unsigned char *src, int nbytes)
413{
414 int err, nwritten;
415 unsigned char compressed;
416
417 /* Catch errors */
418 if (src == NULL || nbytes < 0)
419 return -1;
420
421 /* Write the compression flag */
422 compressed = G_COMPRESSED_NO;
423 if (write(fd, &compressed, 1) != 1) {
424 G_warning(_("Unable to write compression flag"));
425 return -1;
426 }
427
428 /* Now write the data */
429 nwritten = 0;
430 do {
431 err = write(fd, src + nwritten, nbytes - nwritten);
432 if (err > 0)
433 nwritten += err;
434 } while (err > 0 && nwritten < nbytes);
435 if (err <= 0) {
436 if (err == 0)
437 G_warning(_("Unable to write %d bytes: nothing written"), nbytes);
438 else
439 G_warning(_("Unable to write %d bytes: %s"), nbytes,
440 strerror(errno));
441 }
442
443 if (err < 0 || nwritten != nbytes)
444 return -1;
445
446 /* Account for extra compressed flag */
447 nwritten++;
448
449 /* That's all */
450 return nwritten;
451
452} /* G_write_uncompressed() */
453
454/* vim: set softtabstop=4 shiftwidth=4 expandtab: */
void G_free(void *buf)
Free allocated memory.
Definition alloc.c:150
#define NULL
Definition ccmath.h:32
int G_compress(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz, int number)
Definition compress.c:218
char * G_compressor_name(int number)
Definition compress.c:118
int G_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz, int number)
Definition compress.c:233
int G_write_compressed(int fd, unsigned char *src, int nbytes, int number)
Definition compress.c:323
int G_default_compressor(void)
Definition compress.c:126
int G_write_uncompressed(int fd, const unsigned char *src, int nbytes)
Definition compress.c:412
int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes, int number)
Definition compress.c:244
int G_no_compress(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz)
Definition compress.c:155
#define G_COMPRESSED_YES
Definition compress.c:95
int G_compressor_number(char *name)
Definition compress.c:100
#define G_COMPRESSED_NO
Definition compress.c:94
int G_check_compressor(int number)
Definition compress.c:140
int G_no_compress_bound(int src_sz)
Definition compress.c:150
int G_no_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz)
Definition compress.c:176
int G_compress_bound(int src_sz, int number)
Definition compress.c:202
struct compressor_list compressor[]
Definition compress.h:53
double b
void G_fatal_error(const char *msg,...)
Print a fatal error message to stderr.
Definition gis/error.c:159
void G_warning(const char *msg,...)
Print a warning message to stderr.
Definition gis/error.c:203
const char * name
Definition named_colr.c:6
int G_strcasecmp(const char *x, const char *y)
String compare ignoring case (upper or lower)
Definition strings.c:47
SYMBOL * err(FILE *fp, SYMBOL *s, char *msg)