Merge branch 'feature/UpdateRarfile' into dev

CHANGES.md

@@ -3,9 +3,10 @@
 * Update attr 22.2.0 (683d056) to 23.1.0 (67e4ff2)
 * Update Beautiful Soup 4.12.2 to 4.12.2 (30c58a1)
 * Update diskcache 5.6.1 (4d30686) to 5.6.3 (323787f)
-* Update Pytvmaze library 2.0.8 (81888a5) to 2.0.8 (b451391)
-* Update soupsieve 2.4.1 (2e66beb) to 2.5.0 (dc71495)
 * Update hachoir 3.1.2 (f739b43) to 3.2.0 (38d759f)
+* Update Pytvmaze library 2.0.8 (81888a5) to 2.0.8 (b451391)
+* Update Rarfile 4.1a1 (8a72967) to 4.1 (c9140d8)
+* Update soupsieve 2.4.1 (2e66beb) to 2.5.0 (dc71495)
 
 
 ### 3.30.1 (2023-10-02 22:50:00 UTC)

lib/rarfile/rarfile.py

@@ -59,7 +59,7 @@ import sys
 import warnings
 from binascii import crc32, hexlify
 from datetime import datetime, timezone
-from hashlib import blake2s, pbkdf2_hmac, sha1
+from hashlib import blake2s, pbkdf2_hmac, sha1, sha256
 from pathlib import Path
 from struct import Struct, pack, unpack
 from subprocess import DEVNULL, PIPE, STDOUT, Popen
@@ -92,7 +92,7 @@ class AES_CBC_Decrypt:
             self.decrypt = ciph.decryptor().update
 
 
-__version__ = "4.1a1"
+__version__ = "4.1"
 
 # export only interesting items
 __all__ = ["get_rar_version", "is_rarfile", "is_rarfile_sfx", "RarInfo", "RarFile", "RarExtFile"]
@@ -110,6 +110,12 @@ UNAR_TOOL = "unar"
 #: executable for bsdtar tool
 BSDTAR_TOOL = "bsdtar"
 
+#: executable for p7zip/7z tool
+SEVENZIP_TOOL = "7z"
+
+#: executable for alternative 7z tool
+SEVENZIP2_TOOL = "7zz"
+
 #: default fallback charset
 DEFAULT_CHARSET = "windows-1252"
 
@@ -282,6 +288,9 @@ DOS_MODE_SYSTEM = 0x04
 DOS_MODE_HIDDEN = 0x02
 DOS_MODE_READONLY = 0x01
 
+RAR5_PW_CHECK_SIZE = 8
+RAR5_PW_SUM_SIZE = 4
+
 ##
 ## internal constants
 ##
@@ -300,6 +309,8 @@ _BAD_CHARS = r"""\x00-\x1F<>|"?*"""
 RC_BAD_CHARS_UNIX = re.compile(r"[%s]" % _BAD_CHARS)
 RC_BAD_CHARS_WIN32 = re.compile(r"[%s:^\\]" % _BAD_CHARS)
 
+FORCE_TOOL = False
+
 
 def _find_sfx_header(xfile):
     sig = RAR_ID[:-1]
@@ -641,17 +652,6 @@ class RarInfo:
 
 class RarFile:
     """Parse RAR structure, provide access to files in archive.
-    """
-
-    #: File name, if available.  Unicode string or None.
-    filename = None
-
-    #: Archive comment.  Unicode string or None.
-    comment = None
-
-    def __init__(self, file, mode="r", charset=None, info_callback=None,
-                 crc_check=True, errors="stop", part_only=False):
-        """Open and parse a RAR archive.
 
     Parameters:
 
@@ -671,7 +671,18 @@ class RarFile:
         part_only
             If True, read only single file and allow it to be middle-part
             of multi-volume archive.
+
+            .. versionadded:: 4.0
     """
+
+    #: File name, if available.  Unicode string or None.
+    filename = None
+
+    #: Archive comment.  Unicode string or None.
+    comment = None
+
+    def __init__(self, file, mode="r", charset=None, info_callback=None,
+                 crc_check=True, errors="stop", part_only=False):
         if is_filelike(file):
             self.filename = getattr(file, "name", None)
         else:
@@ -751,6 +762,16 @@ class RarFile:
         """
         return self._file_parser.getinfo(name)
 
+    def getinfo_orig(self, name):
+        """Return RarInfo for file source.
+
+        RAR5: if name is hard-linked or copied file,
+        returns original entry with original filename.
+
+        .. versionadded:: 4.1
+        """
+        return self._file_parser.getinfo_orig(name)
+
     def open(self, name, mode="r", pwd=None):
         """Returns file-like object (:class:`RarExtFile`) from where the data can be read.
 
@@ -1058,6 +1079,15 @@ class CommonParser:
         except KeyError:
             raise NoRarEntry("No such file: %s" % fname) from None
 
+    def getinfo_orig(self, member):
+        inf = self.getinfo(member)
+        if inf.file_redir:
+            redir_type, redir_flags, redir_name = inf.file_redir
+            # cannot leave to unrar as it expects copied file to exist
+            if redir_type in (RAR5_XREDIR_FILE_COPY, RAR5_XREDIR_HARD_LINK):
+                inf = self.getinfo(redir_name)
+        return inf
+
     def parse(self):
         """Process file."""
         self._fd = None
@@ -1111,6 +1141,7 @@ class CommonParser:
                     endarc = False
                     self._vol_list.append(volfile)
                     self._main = None
+                    self._hdrenc_main = None
                     continue
                 break
             h.volume = volume
@@ -1255,6 +1286,8 @@ class CommonParser:
             return self._open_unrar(self._rarfile, inf, pwd)
 
     def _open_clear(self, inf):
+        if FORCE_TOOL:
+            return self._open_unrar(self._rarfile, inf)
         return DirectReader(self, inf)
 
     def _open_hack_core(self, inf, pwd, prefix, suffix):
@@ -1302,7 +1335,7 @@ class CommonParser:
         # not giving filename avoids encoding related problems
         fn = None
         if not tmpfile or force_file:
-            fn = inf.filename
+            fn = inf.filename.replace("/", os.path.sep)
 
         # read from unrar pipe
         cmd = setup.open_cmdline(pwd, rarfile, fn)
@@ -1768,14 +1801,18 @@ class RAR5Parser(CommonParser):
     # AES encrypted headers
     _last_aes256_key = (-1, None, None)   # (kdf_count, salt, key)
 
+    def _get_utf8_password(self):
+        pwd = self._password
+        if isinstance(pwd, str):
+            return pwd.encode("utf8")
+        return pwd
+
     def _gen_key(self, kdf_count, salt):
         if self._last_aes256_key[:2] == (kdf_count, salt):
             return self._last_aes256_key[2]
         if kdf_count > 24:
             raise BadRarFile("Too large kdf_count")
-        pwd = self._password
+        pwd = self._get_utf8_password()
-        if isinstance(pwd, str):
-            pwd = pwd.encode("utf8")
         key = pbkdf2_hmac("sha256", pwd, salt, 1 << kdf_count)
         self._last_aes256_key = (kdf_count, salt, key)
         return key
@@ -1938,15 +1975,39 @@ class RAR5Parser(CommonParser):
             h.flags |= RAR_ENDARC_NEXT_VOLUME
         return h
 
+    def _check_password(self, check_value, kdf_count_shift, salt):
+        if len(check_value) != RAR5_PW_CHECK_SIZE + RAR5_PW_SUM_SIZE:
+            return
+
+        hdr_check = check_value[:RAR5_PW_CHECK_SIZE]
+        hdr_sum = check_value[RAR5_PW_CHECK_SIZE:]
+        sum_hash = sha256(hdr_check).digest()
+        if sum_hash[:RAR5_PW_SUM_SIZE] != hdr_sum:
+            return
+
+        kdf_count = (1 << kdf_count_shift) + 32
+        pwd = self._get_utf8_password()
+        pwd_hash = pbkdf2_hmac("sha256", pwd, salt, kdf_count)
+
+        pwd_check = bytearray(RAR5_PW_CHECK_SIZE)
+        len_mask = RAR5_PW_CHECK_SIZE - 1
+        for i, v in enumerate(pwd_hash):
+            pwd_check[i & len_mask] ^= v
+
+        if pwd_check != hdr_check:
+            raise RarWrongPassword()
+
     def _parse_encryption_block(self, h, hdata, pos):
         h.encryption_algo, pos = load_vint(hdata, pos)
         h.encryption_flags, pos = load_vint(hdata, pos)
         h.encryption_kdf_count, pos = load_byte(hdata, pos)
         h.encryption_salt, pos = load_bytes(hdata, 16, pos)
         if h.encryption_flags & RAR5_ENC_FLAG_HAS_CHECKVAL:
-            h.encryption_check_value = load_bytes(hdata, 12, pos)
+            h.encryption_check_value, pos = load_bytes(hdata, 12, pos)
         if h.encryption_algo != RAR5_XENC_CIPHER_AES256:
             raise BadRarFile("Unsupported header encryption cipher")
+        if h.encryption_check_value and self._password:
+            self._check_password(h.encryption_check_value, h.encryption_kdf_count, h.encryption_salt)
         self._hdrenc_main = h
         return h
 
@@ -2183,6 +2244,7 @@ class RarExtFile(io.RawIOBase):
     _remain = 0
     _returncode = 0
     _md_context = None
+    _seeking = False
 
     def _open_extfile(self, parser, inf):
         self.name = inf.filename
@@ -2191,6 +2253,9 @@ class RarExtFile(io.RawIOBase):
 
         if self._fd:
             self._fd.close()
+        if self._seeking:
+            md_class = NoHashContext
+        else:
             md_class = self._inf._md_class or NoHashContext
         self._md_context = md_class()
         self._fd = None
@@ -2282,7 +2347,9 @@ class RarExtFile(io.RawIOBase):
         """
 
         # disable crc check when seeking
+        if not self._seeking:
             self._md_context = NoHashContext()
+            self._seeking = True
 
         fsize = self._inf.file_size
         cur_ofs = self.tell()
@@ -2463,6 +2530,11 @@ class DirectReader(RarExtFile):
     def _read(self, cnt):
         """Read from potentially multi-volume archive."""
 
+        pos = self._fd.tell()
+        need = self._cur.data_offset + self._cur.add_size - self._cur_avail
+        if pos != need:
+            self._fd.seek(need, 0)
+
         buf = []
         while cnt > 0:
             # next vol needed?
@@ -3290,6 +3362,8 @@ class ToolSetup:
     def get_cmdline(self, key, pwd, nodash=False):
         cmdline = list(self.setup[key])
         cmdline[0] = globals()[cmdline[0]]
+        if key == "check_cmd":
+            return cmdline
         self.add_password_arg(cmdline, pwd)
         if not nodash:
             cmdline.append("--")
@@ -3352,10 +3426,30 @@ BSDTAR_CONFIG = {
     "errmap": [None],
 }
 
+SEVENZIP_CONFIG = {
+    "open_cmd": ("SEVENZIP_TOOL", "e", "-so", "-bb0"),
+    "check_cmd": ("SEVENZIP_TOOL", "i"),
+    "password": "-p",
+    "no_password": ("-p",),
+    "errmap": [None,
+               RarWarning, RarFatalError, None, None,           # 1..4
+               None, None, RarUserError, RarMemoryError]        # 5..8
+}
+
+SEVENZIP2_CONFIG = {
+    "open_cmd": ("SEVENZIP2_TOOL", "e", "-so", "-bb0"),
+    "check_cmd": ("SEVENZIP2_TOOL", "i"),
+    "password": "-p",
+    "no_password": ("-p",),
+    "errmap": [None,
+               RarWarning, RarFatalError, None, None,           # 1..4
+               None, None, RarUserError, RarMemoryError]        # 5..8
+}
+
 CURRENT_SETUP = None
 
 
-def tool_setup(unrar=True, unar=True, bsdtar=True, force=False):
+def tool_setup(unrar=True, unar=True, bsdtar=True, sevenzip=True, sevenzip2=True, force=False):
     """Pick a tool, return cached ToolSetup.
     """
     global CURRENT_SETUP
@@ -3368,6 +3462,10 @@ def tool_setup(unrar=True, unar=True, bsdtar=True, force=False):
         lst.append(UNRAR_CONFIG)
     if unar:
         lst.append(UNAR_CONFIG)
+    if sevenzip:
+        lst.append(SEVENZIP_CONFIG)
+    if sevenzip2:
+        lst.append(SEVENZIP2_CONFIG)
     if bsdtar:
         lst.append(BSDTAR_CONFIG)