Flafka Not Able to Read Snappy Compressed
Python snappy.shrink() Examples
The post-obit are 26 lawmaking examples for showing how to utilise snappy.compress() . These examples are extracted from open up source projects. Y'all can vote up the ones yous like or vote down the ones y'all don't similar, and get to the original project or source file by post-obit the links to a higher place each example.
Yous may bank check out the related API usage on the sidebar.
You may also want to check out all available functions/classes of the module snappy , or endeavor the search part 
.
Example 1
def FlushChunk(cocky, clamper): if len(chunk) == 0: return bevy_offset = cocky.bevy_length if self.pinch == lexicon.AFF4_IMAGE_COMPRESSION_ZLIB: compressed_chunk = zlib.compress(chunk) elif (snappy and self.compression == lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY): compressed_chunk = snappy.compress(chunk) elif self.pinch == dictionary.AFF4_IMAGE_COMPRESSION_STORED: compressed_chunk = clamper compressedLen = len(compressed_chunk) if compressedLen < self.chunk_size - 16: self.bevy_index.append((bevy_offset, compressedLen)) self.bevy.append(compressed_chunk) cocky.bevy_length += compressedLen else: self.bevy_index.append((bevy_offset, self.chunk_size)) cocky.bevy.append(chunk) cocky.bevy_length += cocky.chunk_size #self.bevy_index.append((bevy_offset, len(compressed_chunk))) #cocky.bevy.append(compressed_chunk) #self.bevy_length += len(compressed_chunk) self.chunk_count_in_bevy += ane #cocky.buffer = clamper[self.chunk_size:] if self.chunk_count_in_bevy >= self.chunks_per_segment: self._FlushBevy()
Example 2
def compress(self, data): if self._message_encoding == "gzip" or self._message_encoding == "deflate": import zlib render zlib.shrink(information) elif self._message_encoding == "snappy": import snappy return snappy.compress(data) else: raise UnsupportedMessageEncodingError( "Unsupported compression: {}".format(self._message_encoding)) Instance 3
def write_message(cocky, data: bytes, compress=False): if compress: data = self.shrink(data) if len(information) > self._max_message_length: enhance MessageTooLargeError( "Trying to send bulletin larger than max: {message_length} > {max_message_length}".format( message_length=len(information), max_message_length=self._max_message_length, ) ) self._buffer.suspend(struct.pack('>?I', compress, len(data))) self._buffer.append(data) Case 4
def _zlib_no_compress(data): """Compress data with zlib level 0.""" cobj = zlib.compressobj(0) render b"".join([cobj.compress(data), cobj.flush()])
Example five
def compress(data): return snappy.compress(data)
Example vi
def __init__(self, level): # Jython zlib.compress doesn't support -i if level == -i: self.compress = zlib.compress # Jython zlib.compress also doesn't back up 0 elif level == 0: self.compress = _zlib_no_compress else: self.compress = lambda data: zlib.compress(information, level)
Example 7
def compress(self, data): return information
Case 8
def shrink(self, data): buf = io.BytesIO() with gzip.GzipFile(fileobj=buf, mode='wb', compresslevel=1) equally f: f.write(data) return buf.getvalue()
Case 9
def encode(cocky, obj, container): render self._compressor.compress( self._format_encoder.encode(obj, container))
Example ten
def compress(self, data): return snappy.compress(data)
Example xi
def _pack_msgpack_snappy(obj): # print "pack", obj tmp = msgpack.dumps(obj, encoding='utf-8') if len(tmp) > 1000: render b'Due south' + snappy.shrink(tmp) else: render b'\0' + tmp
Case 12
def _zlib_no_compress(data): """Shrink data with zlib level 0.""" cobj = zlib.compressobj(0) render b"".join([cobj.compress(data), cobj.flush()])
Case xiii
def compress(data): return snappy.compress(information)
Example 14
def __init__(self, level): # Jython zlib.compress doesn't support -1 if level == -ane: self.compress = zlib.compress # Jython zlib.compress also doesn't support 0 elif level == 0: self.compress = _zlib_no_compress else: cocky.compress = lambda data: zlib.compress(data, level)
Instance 15
def compress(information): # ZstdCompressor is not thread safety. # TODO: Utilise a pool? return ZstdCompressor().shrink(information)
Example 16
def compress(data): """ Compresses given information via the snappy algorithm. The result is preceded with a header containing the string 'SNAPPY' and the default and min-compat versions (both ``1``). The block size for the compression is hard-coded at 32kb. If ``python-snappy`` is not installed a ``RuntimeError`` is raised. """ if not snappy_available: raise RuntimeError("Snappy compression unavailable.") buff = BytesIO() vitrify.write(raw_header) for block_num in range(0, len(data), BLOCK_SIZE): cake = data[block_num:block_num + BLOCK_SIZE] compressed = snappy.compress(block) buff.write(struct.pack("!i", len(compressed))) vitrify.write(compressed) result = buff.getvalue() vitrify.close() return result Example 17
def _compress_event(cocky, consequence: BaseEvent) -> Union[BaseEvent, bytes]: if cocky.has_snappy_support: import snappy render cast( bytes, snappy.compress(pickle.dumps(outcome, protocol=pickle.HIGHEST_PROTOCOL)), ) else: return event
Example xviii
def _pack_msgpack_snappy(obj): # impress "pack", obj tmp = msgpack.dumps(obj, encoding='utf-8') if len(tmp) > grand: return b'S' + snappy.compress(tmp) else: return b'\0' + tmp
Example 19
def _serialize_gossip(operation: ssz.Serializable) -> bytes: encoded_operation = ssz.encode(operation) return snappy.compress(encoded_operation)
Example 20
def compress(self, data: bytes) -> bytes: return snappy.compress(data)
Example 21
def compress(self, data: bytes) -> bytes: return data
Example 22
def encode(cocky, cmd_id: int, snappy_support: bool) -> MessageAPI: raw_payload_data = self.serialization_codec.encode(self.payload) if snappy_support: payload_data = self.compression_codec.compress(raw_payload_data) else: payload_data = raw_payload_data cmd_id_data = rlp.encode(cmd_id, sedes=rlp.sedes.big_endian_int) frame_size = len(cmd_id_data) + len(payload_data) if frame_size.bit_length() > 24: enhance ValueError("Frame size has to fit in a 3-byte integer") # Frame-size is a 3-bit integer header = frame_size.to_bytes(3, 'big') + RLPX_HEADER_DATA body = cmd_id_data + payload_data return Message(header, trunk) Case 23
def _pack_msgpack_snappy(obj): # impress "pack", obj tmp = msgpack.dumps(obj, encoding='utf-8') if len(tmp) > 1000: render b'S' + snappy.shrink(tmp) else: return b'\0' + tmp
Example 24
def read(self, _): # Cease copying when the bevy is full. if cocky.chunk_count_in_bevy >= self.possessor.chunks_per_segment: return "" chunk = cocky.stream.read(self.owner.chunk_size) if non clamper: render "" self.size += len(chunk) if self.owner.compression == lexicon.AFF4_IMAGE_COMPRESSION_ZLIB: compressed_chunk = zlib.shrink(chunk) elif (snappy and self.owner.pinch == lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY): compressed_chunk = snappy.compress(clamper) elif self.owner.compression == lexicon.AFF4_IMAGE_COMPRESSION_STORED: compressed_chunk = chunk compressedLen = len(compressed_chunk) self.chunk_count_in_bevy += one if compressedLen < self.possessor.chunk_size - 16: self.bevy_index.suspend((self.bevy_length, compressedLen)) self.bevy_length += compressedLen render compressed_chunk else: cocky.bevy_index.append((self.bevy_length, self.owner.chunk_size)) self.bevy_length += self.owner.chunk_size return chunk
Example 25
def snappy_encode(payload, xerial_compatible=False, xerial_blocksize=32 * 1024): """ Compress the given data with the Snappy algorithm. :param bytes payload: Information to shrink. :param bool xerial_compatible: If set and so the stream is broken into length-prefixed blocks in a fashion compatible with the xerial snappy library. The format winds upward being:: +-------------+------------+--------------+------------+--------------+ | Header | Block1_len | Block1 information | BlockN len | BlockN data | |-------------+------------+--------------+------------+--------------| | 16 bytes | Exist int32 | snappy bytes | Exist int32 | snappy bytes | +-------------+------------+--------------+------------+--------------+ :param int xerial_blocksize: Number of bytes per chunk to independently Snappy encode. 32k is the default in the xerial library. :returns: Compressed bytes. :rtype: :class:`bytes` """ if not has_snappy(): # FIXME This should be static, not checked every call. raise NotImplementedError("Snappy codec is non available") if xerial_compatible: def _chunker(): for i in range(0, len(payload), xerial_blocksize): yield payload[i:i+xerial_blocksize] out = BytesIO() out.write(_XERIAL_HEADER) for clamper in _chunker(): cake = snappy.compress(chunk) out.write(struct.pack('!i', len(cake))) out.write(block) out.seek(0) return out.read() else: return snappy.compress(payload) Example 26
def _write_block(cocky): if non cocky._header_written: self._write_header() if cocky.block_count > 0: # write number of items in block self.encoder.write_long(self.block_count) # write cake contents uncompressed_data = self.buffer_writer.getvalue() if self.get_meta(CODEC_KEY) == 'null': compressed_data = uncompressed_data compressed_data_length = len(compressed_data) elif self.get_meta(CODEC_KEY) == 'deflate': # The starting time 2 characters and terminal character are zlib # wrappers around debunk data. compressed_data = zlib.shrink(uncompressed_data)[two:-1] compressed_data_length = len(compressed_data) elif self.get_meta(CODEC_KEY) == 'snappy': compressed_data = snappy.compress(uncompressed_data) compressed_data_length = len(compressed_data) + 4 # crc32 elif self.get_meta(CODEC_KEY) == 'xz': compressed_data = lzma.compress(uncompressed_data, format=lzma.FORMAT_XZ) compressed_data_length = len(compressed_data) else: fail_msg = '"%southward" codec is non supported.' % self.get_meta(CODEC_KEY) raise DataFileException(fail_msg) # Write length of block self.encoder.write_long(compressed_data_length) # Write block cocky.writer.write(compressed_data) # Write CRC32 checksum for Snappy if self.get_meta(CODEC_KEY) == 'snappy': self.encoder.write_crc32(uncompressed_data) # write sync marker self.writer.write(self.sync_marker) # reset buffer self.buffer_writer.truncate(0) self.buffer_writer.seek(0) cocky.block_count = 0
Source: https://www.programcreek.com/python/example/103918/snappy.compress