_roslz4module.c
Go to the documentation of this file.
1 /*********************************************************************
2 * Software License Agreement (BSD License)
3 *
4 * Copyright (c) 2014, Ben Charrow
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
17 * * Neither the name of Willow Garage, Inc. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 ********************************************************************/
34 
35 #include "Python.h"
36 
37 #include "roslz4/lz4s.h"
38 
39 struct module_state {
40  PyObject *error;
41 };
42 
43 #if PY_MAJOR_VERSION >= 3
44 #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m))
45 #else
46 #define GETSTATE(m) (&_state)
47 static struct module_state _state;
48 #endif
49 
50 /* Taken from Python's _bz2module.c */
51 static int
52 grow_buffer(PyObject **buf)
53 {
54  /* Expand the buffer by an amount proportional to the current size,
55  giving us amortized linear-time behavior. Use a less-than-double
56  growth factor to avoid excessive allocation. */
57  size_t size = PyBytes_GET_SIZE(*buf);
58  size_t new_size = size + (size >> 3) + 6;
59  if (new_size > size) {
60  return _PyBytes_Resize(buf, new_size);
61  } else { /* overflow */
62  PyErr_SetString(PyExc_OverflowError,
63  "Unable to allocate buffer - output too large");
64  return -1;
65  }
66 }
67 
68 /*============================== LZ4Compressor ==============================*/
69 
70 typedef struct {
71  PyObject_HEAD
74 
75 static void
77 {
78  roslz4_compressEnd(&self->stream);
79  Py_TYPE(self)->tp_free((PyObject*)self);
80 }
81 
82 static int
83 LZ4Compressor_init(LZ4Compressor *self, PyObject *args, PyObject *kwds)
84 {
85  (void)kwds;
86  if (!PyArg_ParseTuple(args, ":__init__")) {
87  return -1;
88  }
89 
90  int ret = roslz4_compressStart(&self->stream, 6);
91  if (ret != ROSLZ4_OK) {
92  PyErr_SetString(PyExc_RuntimeError, "error initializing roslz4 stream");
93  return -1;
94  }
95  return 0;
96 }
97 
98 static PyObject *
99 compress_impl(LZ4Compressor *self, Py_buffer *input, PyObject *output)
100 {
101  /* Allocate output string */
102  int initial_size = roslz4_blockSizeFromIndex(self->stream.block_size_id) + 64;
103  output = PyBytes_FromStringAndSize(NULL, initial_size);
104  if (!output) {
105  if (input != NULL) { PyBuffer_Release(input); }
106  return NULL;
107  }
108 
109  /* Setup stream */
110  int action;
111  if (input != NULL) {
112  action = ROSLZ4_RUN;
113  self->stream.input_next = input->buf;
114  self->stream.input_left = input->len;
115  } else {
116  action = ROSLZ4_FINISH;
117  self->stream.input_next = NULL;
118  self->stream.input_left = 0;
119  }
120  self->stream.output_next = PyBytes_AS_STRING(output);
121  self->stream.output_left = PyBytes_GET_SIZE(output);
122 
123  /* Compress data */
124  int status;
125  int output_written = 0;
126  while ((action == ROSLZ4_FINISH) ||
127  (action == ROSLZ4_RUN && self->stream.input_left > 0)) {
128  int out_start = self->stream.total_out;
129  status = roslz4_compress(&self->stream, action);
130  output_written += self->stream.total_out - out_start;
131  if (status == ROSLZ4_OK) {
132  continue;
133  } else if (status == ROSLZ4_STREAM_END) {
134  break;
135  } else if (status == ROSLZ4_OUTPUT_SMALL) {
136  if (grow_buffer(&output) < 0) {
137  goto error;
138  }
139  self->stream.output_next = PyBytes_AS_STRING(output) + output_written;
140  self->stream.output_left = PyBytes_GET_SIZE(output) - output_written;
141  } else if (status == ROSLZ4_PARAM_ERROR) {
142  PyErr_SetString(PyExc_IOError, "bad block size parameter");
143  goto error;
144  } else if (status == ROSLZ4_ERROR) {
145  PyErr_SetString(PyExc_IOError, "error compressing");
146  goto error;
147  } else {
148  PyErr_Format(PyExc_RuntimeError, "unhandled return code %i", status);
149  goto error;
150  }
151  }
152 
153  /* Shrink return buffer */
154  if (output_written != PyBytes_GET_SIZE(output)) {
155  _PyBytes_Resize(&output, output_written);
156  }
157 
158  if (input != NULL) { PyBuffer_Release(input); }
159  return output;
160 
161 error:
162  if (input != NULL) { PyBuffer_Release(input); }
163  Py_XDECREF(output);
164  return NULL;
165 }
166 
167 static PyObject *
168 LZ4Compressor_compress(LZ4Compressor *self, PyObject *args)
169 {
170  Py_buffer input;
171  PyObject *output = NULL;
172 
173  /* TODO: Keyword argument */
174  if (!PyArg_ParseTuple(args, "s*:compress", &input)) {
175  return NULL;
176  }
177 
178  return compress_impl(self, &input, output);
179 }
180 
181 static PyObject *
182 LZ4Compressor_flush(LZ4Compressor *self, PyObject *args)
183 {
184  PyObject *output = NULL;
185 
186  if (!PyArg_ParseTuple(args, ":flush")) {
187  return NULL;
188  }
189 
190  return compress_impl(self, NULL, output);
191 }
192 
193 
194 static PyMethodDef LZ4Compressor_methods[] = {
195  {"compress", (PyCFunction)LZ4Compressor_compress, METH_VARARGS, "method doc"},
196  {"flush", (PyCFunction)LZ4Compressor_flush, METH_VARARGS, "method doc"},
197  {NULL} /* Sentinel */
198 };
199 
200 static PyTypeObject LZ4Compressor_Type = {
201  PyVarObject_HEAD_INIT(NULL, 0)
202  "_roslz4.LZ4Compressor", /* tp_name */
203  sizeof(LZ4Compressor), /* tp_basicsize */
204  0, /* tp_itemsize */
205  (destructor)LZ4Compressor_dealloc, /* tp_dealloc */
206  0, /* tp_print */
207  0, /* tp_getattr */
208  0, /* tp_setattr */
209  0, /* tp_compare */
210  0, /* tp_repr */
211  0, /* tp_as_number */
212  0, /* tp_as_sequence */
213  0, /* tp_as_mapping */
214  0, /* tp_hash */
215  0, /* tp_call */
216  0, /* tp_str */
217  0, /* tp_getattro */
218  0, /* tp_setattro */
219  0, /* tp_as_buffer */
220  Py_TPFLAGS_DEFAULT, /* tp_flags */
221  "LZ4Compressor objects", /* tp_doc */
222  0, /* tp_traverse */
223  0, /* tp_clear */
224  0, /* tp_richcompare */
225  0, /* tp_weaklistoffset */
226  0, /* tp_iter */
227  0, /* tp_iternext */
228  LZ4Compressor_methods, /* tp_methods */
229  0, /* tp_members */
230  0, /* tp_getset */
231  0, /* tp_base */
232  0, /* tp_dict */
233  0, /* tp_descr_get */
234  0, /* tp_descr_set */
235  0, /* tp_dictoffset */
236  (initproc)LZ4Compressor_init /* tp_init */
237 };
238 
239 /*============================= LZ4Decompressor =============================*/
240 
241 typedef struct {
242  PyObject_HEAD
245 
246 static void
248 {
249  roslz4_decompressEnd(&self->stream);
250  Py_TYPE(self)->tp_free((PyObject*)self);
251 }
252 
253 static int
254 LZ4Decompressor_init(LZ4Decompressor *self, PyObject *args, PyObject *kwds)
255 {
256  (void)kwds;
257  if (!PyArg_ParseTuple(args, ":__init__")) {
258  return -1;
259  }
260 
261  int ret = roslz4_decompressStart(&self->stream);
262  if (ret != ROSLZ4_OK) {
263  PyErr_SetString(PyExc_RuntimeError, "error initializing roslz4 stream");
264  return -1;
265  }
266  return 0;
267 }
268 
269 static PyObject *
271 {
272  Py_buffer input;
273  PyObject *output = NULL;
274 
275  /* TODO: Keyword argument */
276  if (!PyArg_ParseTuple(args, "s*:decompress", &input)) {
277  return NULL;
278  }
279 
280  /* Allocate 1 output block. If header not read, use compression block size */
281  int block_size;
282  if (self->stream.block_size_id == -1 ) {
283  block_size = roslz4_blockSizeFromIndex(6);
284  } else {
285  block_size = roslz4_blockSizeFromIndex(self->stream.block_size_id);
286  }
287 
288  output = PyBytes_FromStringAndSize(NULL, block_size);
289  if (!output) {
290  PyBuffer_Release(&input);
291  return NULL;
292  }
293 
294  /* Setup stream */
295  self->stream.input_next = input.buf;
296  self->stream.input_left = input.len;
297  self->stream.output_next = PyBytes_AS_STRING(output);
298  self->stream.output_left = PyBytes_GET_SIZE(output);
299 
300  int output_written = 0;
301  while (self->stream.input_left > 0) {
302  int out_start = self->stream.total_out;
303  int status = roslz4_decompress(&self->stream);
304  output_written += self->stream.total_out - out_start;
305  if (status == ROSLZ4_OK) {
306  continue;
307  } else if (status == ROSLZ4_STREAM_END) {
308  break;
309  } else if (status == ROSLZ4_OUTPUT_SMALL) {
310  if (grow_buffer(&output) < 0) {
311  goto error;
312  }
313  self->stream.output_next = PyBytes_AS_STRING(output) + output_written;
314  self->stream.output_left = PyBytes_GET_SIZE(output) - output_written;
315  } else if (status == ROSLZ4_ERROR) {
316  PyErr_SetString(PyExc_IOError, "error decompressing");
317  goto error;
318  } else if (status == ROSLZ4_DATA_ERROR) {
319  PyErr_SetString(PyExc_IOError, "malformed data to decompress");
320  goto error;
321  } else {
322  PyErr_Format(PyExc_RuntimeError, "unhandled return code %i", status);
323  goto error;
324  }
325  }
326 
327  if (output_written != PyBytes_GET_SIZE(output)) {
328  _PyBytes_Resize(&output, output_written);
329  }
330 
331  PyBuffer_Release(&input);
332  return output;
333 
334 error:
335  PyBuffer_Release(&input);
336  Py_XDECREF(output);
337  return NULL;
338 }
339 
340 static PyMethodDef LZ4Decompressor_methods[] = {
341  {"decompress", (PyCFunction)LZ4Decompressor_decompress, METH_VARARGS, "method doc"},
342  {NULL} /* Sentinel */
343 };
344 
345 static PyTypeObject LZ4Decompressor_Type = {
346  PyVarObject_HEAD_INIT(NULL, 0)
347  "_roslz4.LZ4Decompressor", /* tp_name */
348  sizeof(LZ4Decompressor), /* tp_basicsize */
349  0, /* tp_itemsize */
350  (destructor)LZ4Decompressor_dealloc, /* tp_dealloc */
351  0, /* tp_print */
352  0, /* tp_getattr */
353  0, /* tp_setattr */
354  0, /* tp_compare */
355  0, /* tp_repr */
356  0, /* tp_as_number */
357  0, /* tp_as_sequence */
358  0, /* tp_as_mapping */
359  0, /* tp_hash */
360  0, /* tp_call */
361  0, /* tp_str */
362  0, /* tp_getattro */
363  0, /* tp_setattro */
364  0, /* tp_as_buffer */
365  Py_TPFLAGS_DEFAULT, /* tp_flags */
366  "LZ4Decompressor objects", /* tp_doc */
367  0, /* tp_traverse */
368  0, /* tp_clear */
369  0, /* tp_richcompare */
370  0, /* tp_weaklistoffset */
371  0, /* tp_iter */
372  0, /* tp_iternext */
373  LZ4Decompressor_methods, /* tp_methods */
374  0, /* tp_members */
375  0, /* tp_getset */
376  0, /* tp_base */
377  0, /* tp_dict */
378  0, /* tp_descr_get */
379  0, /* tp_descr_set */
380  0, /* tp_dictoffset */
381  (initproc)LZ4Decompressor_init /* tp_init */
382 };
383 
384 
385 /*========================== Module initialization ==========================*/
386 
387 #if PY_MAJOR_VERSION >= 3
388 
389 static int roslz4_traverse(PyObject *m, visitproc visit, void *arg) {
390  Py_VISIT(GETSTATE(m)->error);
391  return 0;
392 }
393 
394 static int roslz4_clear(PyObject *m) {
395  Py_CLEAR(GETSTATE(m)->error);
396  return 0;
397 }
398 
399 static struct PyModuleDef moduledef = {
400  PyModuleDef_HEAD_INIT,
401  "_roslz4",
402  NULL,
403  sizeof(struct module_state),
404  NULL,
405  NULL,
406  roslz4_traverse,
407  roslz4_clear,
408  NULL
409 };
410 #define INITERROR return NULL
411 
412 PyObject *
413 PyInit__roslz4(void)
414 
415 #else
416 #define INITERROR return
417 
418 void
420 #endif
421 {
422  PyObject *m;
423 
424  LZ4Compressor_Type.tp_new = PyType_GenericNew;
425  if (PyType_Ready(&LZ4Compressor_Type) < 0) {
426  INITERROR;
427  }
428 
429  LZ4Decompressor_Type.tp_new = PyType_GenericNew;
430  if (PyType_Ready(&LZ4Decompressor_Type) < 0) {
431  INITERROR;
432  }
433 
434 #if PY_MAJOR_VERSION >= 3
435  m = PyModule_Create(&moduledef);
436 #else
437  m = Py_InitModule("_roslz4", NULL);
438 #endif
439 
440  if (m == NULL) {
441  INITERROR;
442  }
443 
444  Py_INCREF(&LZ4Compressor_Type);
445  PyModule_AddObject(m, "LZ4Compressor", (PyObject *)&LZ4Compressor_Type);
446  Py_INCREF(&LZ4Decompressor_Type);
447  PyModule_AddObject(m, "LZ4Decompressor", (PyObject *)&LZ4Decompressor_Type);
448 
449 #if PY_MAJOR_VERSION >= 3
450  return m;
451 #endif
452 }
void roslz4_compressEnd(roslz4_stream *stream)
Definition: lz4s.c:353
PyObject_HEAD roslz4_stream stream
Definition: _roslz4module.c:72
static void LZ4Decompressor_dealloc(LZ4Decompressor *self)
PyObject_HEAD roslz4_stream stream
int roslz4_blockSizeFromIndex(int block_id)
Definition: lz4s.c:307
static int LZ4Compressor_init(LZ4Compressor *self, PyObject *args, PyObject *kwds)
Definition: _roslz4module.c:83
static PyMethodDef LZ4Compressor_methods[]
int roslz4_compress(roslz4_stream *stream, int action)
Definition: lz4s.c:317
const int ROSLZ4_OUTPUT_SMALL
Definition: lz4s.h:48
PyObject * error
Definition: _roslz4module.c:40
static int LZ4Decompressor_init(LZ4Decompressor *self, PyObject *args, PyObject *kwds)
int roslz4_decompressStart(roslz4_stream *stream)
Definition: lz4s.c:359
static void LZ4Compressor_dealloc(LZ4Compressor *self)
Definition: _roslz4module.c:76
static struct module_state _state
Definition: _roslz4module.c:47
static int grow_buffer(PyObject **buf)
Definition: _roslz4module.c:52
const int ROSLZ4_STREAM_END
Definition: lz4s.h:51
#define INITERROR
const int ROSLZ4_PARAM_ERROR
Definition: lz4s.h:46
void roslz4_decompressEnd(roslz4_stream *str)
Definition: lz4s.c:567
static PyMethodDef LZ4Decompressor_methods[]
static PyTypeObject LZ4Compressor_Type
const int ROSLZ4_DATA_ERROR
Definition: lz4s.h:47
void init_roslz4(void)
const int ROSLZ4_OK
Definition: lz4s.h:50
static PyTypeObject LZ4Decompressor_Type
static PyObject * compress_impl(LZ4Compressor *self, Py_buffer *input, PyObject *output)
Definition: _roslz4module.c:99
static PyObject * LZ4Compressor_compress(LZ4Compressor *self, PyObject *args)
#define GETSTATE(m)
Definition: _roslz4module.c:46
int roslz4_compressStart(roslz4_stream *stream, int block_size_id)
Definition: lz4s.c:311
const int ROSLZ4_FINISH
Definition: lz4s.h:55
int roslz4_decompress(roslz4_stream *stream)
Definition: lz4s.c:536
const int ROSLZ4_ERROR
Definition: lz4s.h:49
static PyObject * LZ4Decompressor_decompress(LZ4Decompressor *self, PyObject *args)
static PyObject * LZ4Compressor_flush(LZ4Compressor *self, PyObject *args)
const int ROSLZ4_RUN
Definition: lz4s.h:54


roslz4
Author(s): Ben Charrow
autogenerated on Mon Nov 2 2020 03:52:18