Permalink
Feb 4, 2000
Apr 30, 1997
Apr 30, 1997
Apr 30, 1997
Nov 3, 2018
May 6, 2012
Jun 4, 2012
Oct 18, 2013
May 6, 2012
Jun 4, 2012
May 6, 2012
Sep 23, 2010
Sep 23, 2010
Apr 10, 2015
May 13, 2019
Apr 30, 1997
Apr 10, 2015
May 29, 2002
May 29, 2002
May 29, 2002
Oct 18, 2013
Nov 11, 2012
Apr 10, 2015
Jun 19, 2012
Nov 16, 2019
Mar 26, 1998
Apr 30, 1997
Apr 10, 2015
Apr 30, 1997
Apr 30, 1997
Apr 30, 1997
Jun 6, 2007
Aug 13, 2007
Jan 21, 2020
Apr 30, 1997
Mar 23, 2015
Mar 23, 2015
Apr 30, 1997
Apr 30, 1997
Apr 30, 1997
Apr 10, 2015
Apr 10, 2015
Apr 30, 1997
Mar 28, 2005
Apr 30, 1997
Jul 27, 2004
Aug 9, 2001
Jan 3, 2010
Apr 10, 2015
Sep 2, 2021
Apr 10, 2015
Apr 10, 2015
Apr 10, 2015
Oct 17, 2022
Apr 10, 2015
Apr 10, 2015
Oct 17, 2022
Apr 10, 2015
Apr 10, 2015
Sep 2, 2021
Sep 2, 2021
Sep 2, 2021
Apr 12, 2022
Aug 17, 2010
Sep 2, 2021
Nov 19, 2021
Sep 2, 2021
Oct 9, 2018
Nov 3, 2018
Oct 9, 2018
Nov 3, 2018
Jan 4, 2009
Nov 3, 2018
Newer
100644
660 lines (554 sloc)
23.2 KB
3
The user of the file doesn't have to worry about the compression,
4
but random access is not allowed."""
5
6
# based on Andrew Kuchling's minigzip.py distributed with the zlib module
7
9
import zlib
13
16
FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
17
18
READ, WRITE = 1, 2
19
20
_COMPRESS_LEVEL_FAST = 1
21
_COMPRESS_LEVEL_TRADEOFF = 6
22
_COMPRESS_LEVEL_BEST = 9
23
28
encoding=None, errors=None, newline=None):
29
"""Open a gzip-compressed file in binary or text mode.
30
31
The filename argument can be an actual filename (a str or bytes object), or
32
an existing file object to read from or write to.
33
34
The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
35
binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
36
"rb", and the default compresslevel is 9.
37
38
For binary mode, this function is equivalent to the GzipFile constructor:
39
GzipFile(filename, mode, compresslevel). In this case, the encoding, errors
40
and newline arguments must not be provided.
41
42
For text mode, a GzipFile object is created, and wrapped in an
43
io.TextIOWrapper instance with the specified encoding, error handling
44
behavior, and line ending(s).
45
46
"""
47
if "t" in mode:
48
if "b" in mode:
49
raise ValueError("Invalid mode: %r" % (mode,))
50
else:
51
if encoding is not None:
52
raise ValueError("Argument 'encoding' not supported in binary mode")
53
if errors is not None:
54
raise ValueError("Argument 'errors' not supported in binary mode")
55
if newline is not None:
56
raise ValueError("Argument 'newline' not supported in binary mode")
60
binary_file = GzipFile(filename, gz_mode, compresslevel)
61
elif hasattr(filename, "read") or hasattr(filename, "write"):
62
binary_file = GzipFile(None, gz_mode, compresslevel, filename)
63
else:
64
raise TypeError("filename must be a str or bytes object, or a file")
65
68
return io.TextIOWrapper(binary_file, encoding, errors, newline)
69
else:
70
return binary_file
71
77
class _PaddedFile:
78
"""Minimal read-only file object that prepends a string to the contents
79
of an actual file. Shouldn't be used outside of gzip.py, as it lacks
80
essential functionality."""
81
82
def __init__(self, f, prepend=b''):
83
self._buffer = prepend
84
self._length = len(prepend)
85
self.file = f
86
self._read = 0
87
88
def read(self, size):
89
if self._read is None:
90
return self.file.read(size)
91
if self._read + size <= self._length:
92
read = self._read
93
self._read += size
94
return self._buffer[read:self._read]
95
else:
96
read = self._read
97
self._read = None
98
return self._buffer[read:] + \
99
self.file.read(size-self._length+read)
100
105
self._read -= len(prepend)
106
return
107
self._length = len(self._buffer)
108
self._read = 0
109
115
def seekable(self):
116
return True # Allows fast-forwarding even in unseekable streams
118
119
class BadGzipFile(OSError):
120
"""Exception raised in some cases for invalid gzip files."""
121
122
131
132
# Overridden with internal file object to be closed, if only a filename
133
# is passed in
138
"""Constructor for the GzipFile class.
139
140
At least one of fileobj and filename must be given a
141
non-trivial value.
142
143
The new class instance is based on fileobj, which can be a regular
145
It defaults to None, in which case filename is opened to provide
146
a file object.
147
148
When fileobj is not None, the filename argument is only used to be
150
filename of the uncompressed file. It defaults to the filename of
151
fileobj, if discernible; otherwise, it defaults to the empty string,
152
and in this case the original filename is not included in the header.
153
154
The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', 'wb', 'x', or
155
'xb' depending on whether the file will be read or written. The default
162
and 9 is slowest and produces the most compression. 0 is no compression
163
at all. The default is 9.
166
to the last modification time field in the stream when compressing.
167
If omitted or None, the current time is used.
178
filename = getattr(fileobj, 'name', '')
179
if not isinstance(filename, (str, bytes)):
194
if origmode is None:
195
import warnings
196
warnings.warn(
197
"GzipFile was opened for writing, but this will "
198
"change in future Python releases. "
199
"Specify the mode argument for opening it for writing.",
200
FutureWarning, 2)
201
self.mode = WRITE
202
self._init_write(filename)
203
self.compress = zlib.compressobj(compresslevel,
216
217
@property
218
def mtime(self):
219
"""Last modification time read from stream, or None"""
220
return self._buffer.raw._last_mtime
221
222
def __repr__(self):
225
226
def _init_write(self, filename):
233
235
self.fileobj.write(b'\037\213') # magic header
236
self.fileobj.write(b'\010') # compression method
238
# RFC 1952 requires the FNAME field to be Latin-1. Do not
239
# include filenames that cannot be represented that way.
255
if compresslevel == _COMPRESS_LEVEL_BEST:
256
xfl = b'\002'
257
elif compresslevel == _COMPRESS_LEVEL_FAST:
258
xfl = b'\004'
259
else:
260
xfl = b'\000'
261
self.fileobj.write(xfl)
265
266
def write(self,data):
276
length = len(data)
277
else:
278
# accept any data that supports the buffer protocol
279
data = memoryview(data)
280
length = data.nbytes
282
if length > 0:
283
self.fileobj.write(self.compress.compress(data))
284
self.size += length
287
296
321
def close(self):
325
self.fileobj = None
326
try:
327
if self.mode == WRITE:
328
fileobj.write(self.compress.flush())
329
write32u(fileobj, self.crc)
334
finally:
335
myfileobj = self.myfileobj
336
if myfileobj:
337
self.myfileobj = None
338
myfileobj.close()
339
343
# Ensure the compressor's buffer is flushed
344
self.fileobj.write(self.compress.flush(zlib_mode))
346
347
def fileno(self):
348
"""Invoke the underlying file object's fileno() method.
349
350
This will raise AttributeError if the underlying file object
351
doesn't support fileno().
352
"""
353
return self.fileobj.fileno()
354
355
def rewind(self):
356
'''Return the uncompressed stream file position indicator to the
362
def readable(self):
363
return self.mode == READ
364
365
def writable(self):
366
return self.mode == WRITE
367
368
def seekable(self):
369
return True
370
373
if whence != io.SEEK_SET:
374
if whence == io.SEEK_CUR:
375
offset = self.offset + offset
376
else:
377
raise ValueError('Seek from end not supported')
396
def _read_exact(fp, n):
397
'''Read exactly *n* bytes from `fp`
398
399
This method is required because fp may be unbuffered,
400
i.e. return short reads.
401
'''
402
data = fp.read(n)
403
while len(data) < n:
404
b = fp.read(n - len(data))
405
if not b:
406
raise EOFError("Compressed file ended before the "
407
"end-of-stream marker was reached")
408
data += b
409
return data
410
411
412
def _read_gzip_header(fp):
413
'''Read a gzip header from `fp` and progress to the end of the header.
414
415
Returns last mtime if header was present or None otherwise.
416
'''
417
magic = fp.read(2)
418
if magic == b'':
419
return None
420
421
if magic != b'\037\213':
422
raise BadGzipFile('Not a gzipped file (%r)' % magic)
423
424
(method, flag, last_mtime) = struct.unpack("<BBIxx", _read_exact(fp, 8))
425
if method != 8:
426
raise BadGzipFile('Unknown compression method')
427
428
if flag & FEXTRA:
429
# Read & discard the extra field, if present
430
extra_len, = struct.unpack("<H", _read_exact(fp, 2))
431
_read_exact(fp, extra_len)
432
if flag & FNAME:
433
# Read and discard a null-terminated string containing the filename
434
while True:
435
s = fp.read(1)
436
if not s or s==b'\000':
437
break
438
if flag & FCOMMENT:
439
# Read and discard a null-terminated string containing a comment
440
while True:
441
s = fp.read(1)
442
if not s or s==b'\000':
443
break
444
if flag & FHCRC:
445
_read_exact(fp, 2) # Read & discard the 16-bit header CRC
446
return last_mtime
447
448
452
wbits=-zlib.MAX_WBITS)
453
# Set flag indicating start of a new member
454
self._new_member = True
455
self._last_mtime = None
456
457
def _init_read(self):
459
self._stream_size = 0 # Decompressed size of unconcatenated stream
460
461
def _read_gzip_header(self):
470
return self.readall()
471
# size=0 is special because decompress(max_length=0) is not supported
472
if not size:
473
return b""
474
475
# For certain input data, a single
476
# call to decompress() may not return
477
# any data. In this case, retry until we get some data or reach EOF.
478
while True:
479
if self._decompressor.eof:
480
# Ending case: we've come to the end of a member in the file,
481
# so finish up this member, and read a new gzip header.
482
# Check the CRC and file size, and set the flag so we read
483
# a new member
484
self._read_eof()
485
self._new_member = True
486
self._decompressor = self._decomp_factory(
487
**self._decomp_args)
488
489
if self._new_member:
490
# If the _new_member flag is set, we have to
491
# jump to the next member, if there is one.
492
self._init_read()
493
if not self._read_gzip_header():
494
self._size = self._pos
495
return b""
496
self._new_member = False
497
498
# Read a chunk of data from the file
499
if self._decompressor.needs_input:
500
buf = self._fp.read(READ_BUFFER_SIZE)
501
uncompress = self._decompressor.decompress(buf, size)
502
else:
503
uncompress = self._decompressor.decompress(b"", size)
506
# Prepend the already read bytes to the fileobj so they can
507
# be seen by _read_eof() and _read_gzip_header()
508
self._fp.prepend(self._decompressor.unused_data)
509
510
if uncompress != b"":
512
if buf == b"":
513
raise EOFError("Compressed file ended before the "
514
"end-of-stream marker was reached")
516
self._crc = zlib.crc32(uncompress, self._crc)
517
self._stream_size += len(uncompress)
524
# uncompressed data matches the stored values. Note that the size
525
# stored is the true file size mod 2**32.
532
533
# Gzip files can be padded with zeroes and still have archives.
534
# Consume all zero bytes and set the file position to the first
535
# non-zero byte. See http://www.gzip.org/#faq8
536
c = b"\x00"
537
while c == b"\x00":
538
c = self._fp.read(1)
539
if c:
540
self._fp.prepend(c)
541
542
def _rewind(self):
543
super()._rewind()
544
self._new_member = True
546
547
def _create_simple_gzip_header(compresslevel: int,
548
mtime = None) -> bytes:
549
"""
550
Write a simple gzip header with no extra fields.
551
:param compresslevel: Compresslevel used to determine the xfl bytes.
552
:param mtime: The mtime (must support conversion to a 32-bit integer).
553
:return: A bytes object representing the gzip header.
554
"""
555
if mtime is None:
556
mtime = time.time()
557
if compresslevel == _COMPRESS_LEVEL_BEST:
558
xfl = 2
559
elif compresslevel == _COMPRESS_LEVEL_FAST:
560
xfl = 4
561
else:
562
xfl = 0
563
# Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra
564
# fields added to header), mtime, xfl and os (255 for unknown OS).
565
return struct.pack("<BBBBLBB", 0x1f, 0x8b, 8, 0, int(mtime), xfl, 255)
566
567
570
571
compresslevel sets the compression level in range of 0-9.
572
mtime can be used to set the modification time. The modification time is
573
set to the current time by default.
575
if mtime == 0:
576
# Use zlib as it creates the header with 0 mtime by default.
577
# This is faster and with less overhead.
578
return zlib.compress(data, level=compresslevel, wbits=31)
579
header = _create_simple_gzip_header(compresslevel, mtime)
580
trailer = struct.pack("<LL", zlib.crc32(data), (len(data) & 0xffffffff))
581
# Wbits=-15 creates a raw deflate block.
582
return (header + zlib.compress(data, level=compresslevel, wbits=-15) +
583
trailer)
585
586
def decompress(data):
587
"""Decompress a gzip compressed string in one shot.
588
Return the decompressed string.
589
"""
590
decompressed_members = []
591
while True:
592
fp = io.BytesIO(data)
593
if _read_gzip_header(fp) is None:
594
return b"".join(decompressed_members)
595
# Use a zlib raw deflate compressor
596
do = zlib.decompressobj(wbits=-zlib.MAX_WBITS)
597
# Read all the data except the header
598
decompressed = do.decompress(data[fp.tell():])
599
if not do.eof or len(do.unused_data) < 8:
600
raise EOFError("Compressed file ended before the end-of-stream "
601
"marker was reached")
602
crc, length = struct.unpack("<II", do.unused_data[:8])
603
if crc != zlib.crc32(decompressed):
604
raise BadGzipFile("CRC check failed")
605
if length != (len(decompressed) & 0xffffffff):
606
raise BadGzipFile("Incorrect length of data produced")
607
decompressed_members.append(decompressed)
608
data = do.unused_data[8:].lstrip(b"\x00")
611
def main():
612
from argparse import ArgumentParser
613
parser = ArgumentParser(description=
614
"A simple command line interface for the gzip module: act like gzip, "
615
"but do not delete the input file.")
616
group = parser.add_mutually_exclusive_group()
617
group.add_argument('--fast', action='store_true', help='compress faster')
618
group.add_argument('--best', action='store_true', help='compress better')
619
group.add_argument("-d", "--decompress", action="store_true",
622
parser.add_argument("args", nargs="*", default=["-"], metavar='file')
623
args = parser.parse_args()
624
625
compresslevel = _COMPRESS_LEVEL_TRADEOFF
626
if args.fast:
627
compresslevel = _COMPRESS_LEVEL_FAST
628
elif args.best:
629
compresslevel = _COMPRESS_LEVEL_BEST
630
634
f = GzipFile(filename="", mode="rb", fileobj=sys.stdin.buffer)
635
g = sys.stdout.buffer
644
g = GzipFile(filename="", mode="wb", fileobj=sys.stdout.buffer,
645
compresslevel=compresslevel)