#!/usr/bin/env python # coding=utf-8 # # This file is part of aDBa. # # aDBa is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # aDBa is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with aDBa. If not, see <http://www.gnu.org/licenses/>. import hashlib import logging import os import pickle import requests import sys import time from _23 import etree logger = logging.getLogger('adba') logger.addHandler(logging.NullHandler()) # http://www.radicand.org/blog/orz/2010/2/21/edonkey2000-hash-in-python/ def get_ED2K(filePath, forceHash=False, cacheLocation=os.path.normpath(sys.path[0] + os.sep + "ED2KCache.pickle")): """ Returns the ed2k hash of a given file.""" if not filePath: return None md4 = hashlib.new('md4').copy ed2k_chunk_size = 9728000 try: get_ED2K.ED2KCache except: if os.path.isfile(cacheLocation): with open(cacheLocation, 'rb') as f: get_ED2K.ED2KCache = pickle.load(f) else: get_ED2K.ED2KCache = {} def gen(f): while True: x = f.read(ed2k_chunk_size) if x: yield x else: return def md4_hash(data): m = md4() m.update(data) return m def writeCacheToDisk(): try: if len(get_ED2K.ED2KCache) != 0: with open(cacheLocation, 'wb') as f: pickle.dump(get_ED2K.ED2KCache, f, pickle.HIGHEST_PROTOCOL) except: logger.error("Error occurred while writing back to disk") return file_modified_time = os.path.getmtime(filePath) file_name = os.path.basename(filePath) try: cached_file_modified_time = get_ED2K.ED2KCache[file_name][1] except: # if not existing in cache it will be caught by other test cached_file_modified_time = file_modified_time if forceHash or file_modified_time > cached_file_modified_time or file_name not in get_ED2K.ED2KCache: with open(filePath, 'rb') as f: file_size = os.path.getsize(filePath) # if file size is small enough the ed2k hash is the same as the md4 hash if file_size <= ed2k_chunk_size: full_file = f.read() new_hash = md4_hash(full_file).hexdigest() else: a = gen(f) hashes = [md4_hash(data).digest() for data in a] combinedhash = bytearray() for hash in hashes: combinedhash.extend(hash) new_hash = md4_hash(combinedhash).hexdigest() get_ED2K.ED2KCache[file_name] = (new_hash, file_modified_time) writeCacheToDisk() return new_hash else: return get_ED2K.ED2KCache[file_name][0] def get_file_size(path): size = os.path.getsize(path) return size def read_anidb_xml(file_path=None): if not file_path: file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "animetitles.xml") elif not file_path.endswith("xml"): file_path = os.path.join(file_path, "animetitles.xml") return read_xml_into_etree(file_path) def read_tvdb_map_xml(file_path=None): if not file_path: file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "anime-list.xml") elif not file_path.endswith(".xml"): file_path = os.path.join(file_path, "anime-list.xml") return read_xml_into_etree(file_path) def read_xml_into_etree(filePath): if not filePath: return None if not os.path.isfile(filePath): if not get_anime_titles_xml(filePath): return else: mtime = os.path.getmtime(filePath) if time.time() > mtime + 24 * 60 * 60: if not get_anime_titles_xml(filePath): return xml_a_setree = etree.ElementTree().parse(filePath) return xml_a_setree def _remove_file_failed(file): try: os.remove(file) except OSError: logger.warning("Error occurred while trying to remove file %s" % file) def download_file(url, filename): try: r = requests.get(url, stream=True, verify=False) r.raise_for_status() with open(filename, 'wb') as fp: for chunk in r.iter_content(chunk_size=1024): if chunk: fp.write(chunk) fp.flush() except (requests.HTTPError, requests.exceptions.RequestException): _remove_file_failed(filename) return False except (requests.ConnectionError, requests.Timeout): return False return True def get_anime_titles_xml(path): # return download_file("https://raw.githubusercontent.com/ScudLee/anime-lists/master/animetitles.xml", path) return download_file("https://raw.githubusercontent.com/Anime-Lists/anime-lists/master/animetitles.xml", path) def get_anime_list_xml(path): # return download_file("https://raw.githubusercontent.com/ScudLee/anime-lists/master/anime-list.xml", path) return download_file("https://raw.githubusercontent.com/Anime-Lists/anime-lists/master/anime-list.xml", path)