From d96ad1118691003506c9b666af7bd93514296916 Mon Sep 17 00:00:00 2001 From: Cody Logan Date: Mon, 6 Dec 2021 16:19:03 -0800 Subject: Consistent message logging Use Python's logging faciility for messages instead of printing to stdout (except for download progress bars). --- wikiget/dl.py | 81 ++++++++++++++++++++++++------------------------------ wikiget/wikiget.py | 42 +++++++++++++++++++++------- 2 files changed, 68 insertions(+), 55 deletions(-) diff --git a/wikiget/dl.py b/wikiget/dl.py index 0ac8fec..856d8ca 100644 --- a/wikiget/dl.py +++ b/wikiget/dl.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with Wikiget. If not, see . +import logging import os import sys from urllib.parse import unquote, urlparse @@ -33,10 +34,10 @@ def download(dl, args): if url.netloc: filename = url.path site_name = url.netloc - if args.site is not DEFAULT_SITE and not args.quiet: + if args.site is not DEFAULT_SITE: # this will work even if the user specifies 'commons.wikimedia.org' - print('Warning: target is a URL, ' - 'ignoring site specified with --site') + logging.warning("target is a URL, " + "ignoring site specified with --site") else: filename = dl site_name = args.site @@ -49,19 +50,17 @@ def download(dl, args): filename = file_match.group(2) else: # no file extension and/or prefix, probably an article - print(f"Could not parse input '{filename}' as a file. ") + logging.error(f"Could not parse input '{filename}' as a file.") sys.exit(1) filename = unquote(filename) # remove URL encoding for special characters dest = args.output or filename - if args.verbose >= 2: - print(f'User agent: {USER_AGENT}') + logging.debug(f"User agent: {USER_AGENT}") # connect to site and identify ourselves - if args.verbose >= 1: - print(f'Site name: {site_name}') + logging.info(f"Site name: {site_name}") try: site = Site(site_name, path=args.path, clients_useragent=USER_AGENT) if args.username and args.password: @@ -69,24 +68,22 @@ def download(dl, args): except ConnectionError as e: # usually this means there is no such site, or there's no network # connection, though it could be a certificate problem - print("Error: couldn't connect to specified site.") - if args.verbose >= 2: - print('Full error message:') - print(e) + logging.error("Couldn't connect to specified site.") + logging.debug("Full error message:") + logging.debug(e) sys.exit(1) except HTTPError as e: # most likely a 403 forbidden or 404 not found error for api.php - print("Error: couldn't find the specified wiki's api.php. " - "Check the value of --path.") - if args.verbose >= 2: - print('Full error message:') - print(e) + logging.error("Couldn't find the specified wiki's api.php. " + "Check the value of --path.") + logging.debug("Full error message:") + logging.debug(e) sys.exit(1) except (InvalidResponse, LoginError) as e: # InvalidResponse: site exists, but we couldn't communicate with the # API endpoint for some reason other than an HTTP error. # LoginError: missing or invalid credentials - print(e) + logging.error(e) sys.exit(1) # get info about the target file @@ -95,12 +92,11 @@ def download(dl, args): except APIError as e: # an API error at this point likely means access is denied, # which could happen with a private wiki - print('Error: access denied. Try providing credentials with ' - '--username and --password.') - if args.verbose >= 2: - print('Full error message:') - for i in e.args: - print(i) + logging.error("Access denied. Try providing credentials with " + "--username and --password.") + logging.debug("Full error message:") + for i in e.args: + logging.debug(i) sys.exit(1) if file.imageinfo != {}: @@ -110,26 +106,22 @@ def download(dl, args): file_size = file.imageinfo['size'] file_sha1 = file.imageinfo['sha1'] - if args.verbose >= 1: - print(f"Info: downloading '{filename}' " - f"({file_size} bytes) from {site.host}", - end='') - if args.output: - print(f" to '{dest}'") - else: - print('\n', end='') - print(f'Info: {file_url}') + filename_log = f"Downloading '{filename}' ({file_size} bytes) from {site.host}" + if args.output: + filename_log += f" to '{dest}'" + logging.info(filename_log) + logging.info(f"{file_url}") if os.path.isfile(dest) and not args.force: - print(f"File '{dest}' already exists, skipping download " - "(use -f to ignore)") + logging.warning(f"File '{dest}' already exists, skipping download " + "(use -f to ignore)") else: try: fd = open(dest, 'wb') except IOError as e: - print('File could not be written. ' - 'The following error was encountered:') - print(e) + logging.error("File could not be written. " + "The following error was encountered:") + logging.error(e) sys.exit(1) else: # download the file(s) @@ -150,18 +142,17 @@ def download(dl, args): # verify file integrity and optionally print details dl_sha1 = verify_hash(dest) - if args.verbose >= 1: - print(f'Info: downloaded file SHA1 is {dl_sha1}') - print(f'Info: server file SHA1 is {file_sha1}') + logging.info(f"Downloaded file SHA1 is {dl_sha1}") + logging.info(f"Server file SHA1 is {file_sha1}") if dl_sha1 == file_sha1: - if args.verbose >= 1: - print('Info: hashes match!') + logging.info("Hashes match!") # at this point, we've successfully downloaded the file else: - print('Error: hash mismatch! Downloaded file may be corrupt.') + logging.error("Hash mismatch! Downloaded file may be corrupt.") sys.exit(1) else: # no file information returned - print(f"Target '{filename}' does not appear to be a valid file.") + logging.error(f"Target '{filename}' does not appear to be " + "a valid file.") sys.exit(1) diff --git a/wikiget/wikiget.py b/wikiget/wikiget.py index 1e2e9ed..dfc6027 100644 --- a/wikiget/wikiget.py +++ b/wikiget/wikiget.py @@ -81,29 +81,51 @@ def main(): args = parser.parse_args() - # print API and debug messages in verbose mode + loglevel = logging.WARNING if args.verbose >= 2: - logging.basicConfig(level=logging.DEBUG) + # this includes API and library messages + loglevel = logging.DEBUG elif args.verbose >= 1: - logging.basicConfig(level=logging.WARNING) + loglevel = logging.INFO + elif args.quiet: + loglevel = logging.ERROR + + # set up logger + # TODO: optionally save to log file + logging.basicConfig( + level=loglevel, + # format="%(asctime)s [%(levelname)s] %(message)s" + format="[%(levelname)s] %(message)s" + ) if args.batch: # batch download mode input_file = args.FILE - if args.verbose >= 1: - print(f"Info: using batch file '{input_file}'") + dl_list = [] + + logging.info(f"Using batch file '{input_file}'.") + try: fd = open(input_file, 'r') except IOError as e: - print('File could not be read. ' - 'The following error was encountered:') - print(e) + logging.error("File could not be read. " + "The following error was encountered:") + logging.error(e) sys.exit(1) else: with fd: + # store file contents in memory in case something + # happens to the file while we're downloading for _, line in enumerate(fd): - line = line.strip() - download(line, args) + dl_list.append(line) + + # TODO: validate file contents before download process starts + for line_num, url in enumerate(dl_list, start=1): + url = url.strip() + # keep track of batch file line numbers for + # debugging/logging purposes + logging.info(f"Downloading file {line_num} ({url}):") + download(url, args) else: # single download mode dl = args.FILE -- cgit v1.2.3 From 3b757513dc68a9f846f2d120c3919fb46a89e979 Mon Sep 17 00:00:00 2001 From: Cody Logan Date: Mon, 6 Dec 2021 16:36:15 -0800 Subject: Initial attempt at logging to file --- wikiget/wikiget.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/wikiget/wikiget.py b/wikiget/wikiget.py index dfc6027..4098e03 100644 --- a/wikiget/wikiget.py +++ b/wikiget/wikiget.py @@ -78,6 +78,8 @@ def main(): help='treat FILE as a textfile containing ' 'multiple files to download, one URL or ' 'filename per line', action='store_true') + parser.add_argument('-l', '--logfile', default='', + help='save log output to LOGFILE') args = parser.parse_args() @@ -91,12 +93,17 @@ def main(): loglevel = logging.ERROR # set up logger - # TODO: optionally save to log file - logging.basicConfig( - level=loglevel, - # format="%(asctime)s [%(levelname)s] %(message)s" - format="[%(levelname)s] %(message)s" - ) + if args.logfile: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + filename=args.logfile + ) + else: + logging.basicConfig( + level=loglevel, + format="[%(levelname)s] %(message)s" + ) if args.batch: # batch download mode -- cgit v1.2.3 From 10268e7a76dfe72063d682e6043891b967cbad39 Mon Sep 17 00:00:00 2001 From: Cody Logan Date: Tue, 7 Dec 2021 15:12:41 -0800 Subject: Different log levels for file and console --- wikiget/wikiget.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/wikiget/wikiget.py b/wikiget/wikiget.py index 4098e03..6a537ba 100644 --- a/wikiget/wikiget.py +++ b/wikiget/wikiget.py @@ -92,19 +92,37 @@ def main(): elif args.quiet: loglevel = logging.ERROR - # set up logger + # configure logging: + # console log level is set via -v, -vv, and -q options + # file log level is always info (TODO: add debug option) if args.logfile: + # log to console and file logging.basicConfig( level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", + format="%(asctime)s [%(levelname)-7s] %(message)s", filename=args.logfile ) + + console = logging.StreamHandler() + # TODO: even when loglevel is set to logging.DEBUG, + # debug messages aren't printing to console + console.setLevel(loglevel) + console.setFormatter( + logging.Formatter("[%(levelname)s] %(message)s") + ) + logging.getLogger("").addHandler(console) else: + # log only to console logging.basicConfig( level=loglevel, format="[%(levelname)s] %(message)s" ) + # log events are appended to the file if it already exists, + # so note the start of a new download session + logging.info(f"Starting download session using wikiget {wikiget_version}") + # logging.info(f"Log level is set to {loglevel}") + if args.batch: # batch download mode input_file = args.FILE @@ -131,7 +149,7 @@ def main(): url = url.strip() # keep track of batch file line numbers for # debugging/logging purposes - logging.info(f"Downloading file {line_num} ({url}):") + logging.info(f"Downloading '{url}' at line {line_num}:") download(url, args) else: # single download mode -- cgit v1.2.3 From 3e57a1902f7bf6884662fb2aca403e13787c2d26 Mon Sep 17 00:00:00 2001 From: Cody Logan Date: Tue, 7 Dec 2021 15:30:18 -0800 Subject: Update README with logging info --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 575bb05..53061b6 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,11 @@ access) are also supported with the use of the `--username` and `--password` flags. More detailed information, such as the site used and full URL of the file, can -be displayed with `-v` or `--verbose`. Use `-vv` to display even more detail. -`-q` can be used to silence warnings. +be displayed with `-v` or `--verbose`. Use `-vv` to display even more detail, +mainly debugging information or API messages. `-q` can be used to silence warnings. +A logfile can be specified with `-l` or `--logfile`. If this option is present, the +logfile will contain the same information as `-v` along with timestamps. New log +entries will be appended to an existing logfile. By default, the program won't overwrite existing files with the same name as the target, but this can be forced with `-f` or `--force`. Additionally, the file @@ -55,6 +58,7 @@ wikiget https://en.wikipedia.org/wiki/File:Example.jpg -o test.jpg ## Future plans +- download multiple files at once in batch mode - continue batch download even if input is malformed or file doesn't exist (possibly by raising exceptions in `download()`) - batch download by (Commons) category or user uploads @@ -62,7 +66,7 @@ wikiget https://en.wikipedia.org/wiki/File:Example.jpg -o test.jpg ## Contributing -Pull requests or bug reports are more than welcome. +Pull requests, bug reports, or feature requests are more than welcome. It's recommended that you use a [virtual environment manager](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/) -- cgit v1.2.3 From bb0bf8f0c79c31114a615cb201505de3fae15044 Mon Sep 17 00:00:00 2001 From: Cody Logan Date: Tue, 7 Dec 2021 15:30:56 -0800 Subject: Standardize on double quotes --- setup.py | 74 ++++++++++++++++++++++++------------------------ test/test_validations.py | 32 ++++++++++----------- wikiget/__init__.py | 10 +++---- wikiget/dl.py | 13 +++++---- wikiget/validations.py | 6 ++-- wikiget/version.py | 2 +- wikiget/wikiget.py | 64 ++++++++++++++++++++--------------------- 7 files changed, 101 insertions(+), 100 deletions(-) diff --git a/setup.py b/setup.py index ab809e2..a10c111 100644 --- a/setup.py +++ b/setup.py @@ -23,56 +23,56 @@ from os import path from setuptools import setup, find_packages here = path.abspath(path.dirname(__file__)) -with open(path.join(here, 'README.md'), 'r') as fr: +with open(path.join(here, "README.md"), "r") as fr: long_description = fr.read() version_file = {} -with open(path.join(here, 'wikiget', 'version.py'), 'r') as fv: +with open(path.join(here, "wikiget", "version.py"), "r") as fv: exec(fv.read(), version_file) setup( - name='wikiget', - version=version_file['__version__'], - author='Cody Logan', - author_email='clpo13@gmail.com', - description='CLI tool for downloading files from MediaWiki sites', + name="wikiget", + version=version_file["__version__"], + author="Cody Logan", + author_email="clpo13@gmail.com", + description="CLI tool for downloading files from MediaWiki sites", long_description=long_description, - long_description_content_type='text/markdown', - url='https://github.com/clpo13/wikiget', - keywords='commons download mediawiki wikimedia wikipedia', + long_description_content_type="text/markdown", + url="https://github.com/clpo13/wikiget", + keywords="commons download mediawiki wikimedia wikipedia", packages=find_packages(), classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: End Users/Desktop', - 'License :: OSI Approved :: GNU General Public License v3 or later ' - '(GPLv3+)', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Topic :: Internet', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Multimedia', - 'Topic :: Multimedia :: Graphics', - 'Topic :: Multimedia :: Sound/Audio', - 'Topic :: Multimedia :: Video', - 'Topic :: Utilities', + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: GNU General Public License v3 or later " + "(GPLv3+)", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Topic :: Internet", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Multimedia", + "Topic :: Multimedia :: Graphics", + "Topic :: Multimedia :: Sound/Audio", + "Topic :: Multimedia :: Video", + "Topic :: Utilities", ], - python_requires='>=3.6', - install_requires=['mwclient>=0.10.0', 'requests', 'tqdm'], - setup_requires=['pytest-runner'], - tests_require=['pytest', 'pytest-cov'], + python_requires=">=3.6", + install_requires=["mwclient>=0.10.0", "requests", "tqdm"], + setup_requires=["pytest-runner"], + tests_require=["pytest", "pytest-cov"], project_urls={ - 'Bug Reports': 'https://github.com/clpo13/wikiget/issues', + "Bug Reports": "https://github.com/clpo13/wikiget/issues", }, entry_points={ - 'console_scripts': [ - 'wikiget=wikiget.wikiget:main', + "console_scripts": [ + "wikiget=wikiget.wikiget:main", ], }, ) diff --git a/test/test_validations.py b/test/test_validations.py index 5b7d4fc..8dd4d6d 100644 --- a/test/test_validations.py +++ b/test/test_validations.py @@ -23,8 +23,8 @@ def test_invalid_site_input(): """ Invalid site strings should not return regex match objects. """ - invalid_input = ['example.com', 'vim.wikia.com', - 'en.wikipedia.com', 'en.wikimpedia.org'] + invalid_input = ["example.com", "vim.wikia.com", + "en.wikipedia.com", "en.wikimpedia.org"] for i in invalid_input: site_match = valid_site(i) assert site_match is None @@ -34,8 +34,8 @@ def test_valid_site_input(): """ Valid site strings should return regex match objects. """ - valid_input = ['en.wikipedia.org', 'commons.wikimedia.org', - 'de.wikipedia.org', 'meta.wikimedia.org'] + valid_input = ["en.wikipedia.org", "commons.wikimedia.org", + "de.wikipedia.org", "meta.wikimedia.org"] for i in valid_input: site_match = valid_site(i) assert site_match is not None @@ -46,20 +46,20 @@ def test_file_regex(): File regex should return a match object with match groups corresponding to the file prefix and name. """ - i = 'File:Example.jpg' + i = "File:Example.jpg" file_match = valid_file(i) assert file_match is not None - assert file_match.group(0) == 'File:Example.jpg' # entire match - assert file_match.group(1) == 'File:' # first group - assert file_match.group(2) == 'Example.jpg' # second group + assert file_match.group(0) == "File:Example.jpg" # entire match + assert file_match.group(1) == "File:" # first group + assert file_match.group(2) == "Example.jpg" # second group def test_invalid_file_input(): """ Invalid file strings should not return regex match objects. """ - invalid_input = ['file:example', 'example.jpg', 'Foo Bar.gif', - 'Fil:Example.jpg'] + invalid_input = ["file:example", "example.jpg", "Foo Bar.gif", + "Fil:Example.jpg"] for i in invalid_input: file_match = valid_file(i) assert file_match is None @@ -69,9 +69,9 @@ def test_valid_file_input(): """ Valid file strings should return regex match objects. """ - valid_input = ['Image:example.jpg', 'file:example.jpg', - 'File:example.file-01.jpg', 'FILE:FOO.BMP', - 'File:ß handwritten sample.gif', 'File:A (1).jpeg'] + valid_input = ["Image:example.jpg", "file:example.jpg", + "File:example.file-01.jpg", "FILE:FOO.BMP", + "File:ß handwritten sample.gif", "File:A (1).jpeg"] for i in valid_input: file_match = valid_file(i) assert file_match is not None @@ -81,9 +81,9 @@ def test_verify_hash(tmp_path): """ Confirm that verify_hash returns the proper SHA1 hash. """ - file_name = 'testfile' - file_contents = 'foobar' - file_sha1 = '8843d7f92416211de9ebb963ff4ce28125932878' + file_name = "testfile" + file_contents = "foobar" + file_sha1 = "8843d7f92416211de9ebb963ff4ce28125932878" tmp_file = tmp_path / file_name tmp_file.write_text(file_contents) diff --git a/wikiget/__init__.py b/wikiget/__init__.py index 8437ebf..4adcae3 100644 --- a/wikiget/__init__.py +++ b/wikiget/__init__.py @@ -1,5 +1,5 @@ # wikiget - CLI tool for downloading files from Wikimedia sites -# Copyright (C) 2018, 2019, 2020 Cody Logan and contributors +# Copyright (C) 2018-2021 Cody Logan and contributors # SPDX-License-Identifier: GPL-3.0-or-later # # Wikiget is free software: you can redistribute it and/or modify @@ -22,7 +22,7 @@ from .version import __version__ as wikiget_version # set some global constants BLOCKSIZE = 65536 CHUNKSIZE = 1024 -DEFAULT_SITE = 'commons.wikimedia.org' -DEFAULT_PATH = '/w/' -USER_AGENT = ('wikiget/{} (https://github.com/clpo13/wikiget) ' - 'mwclient/{}'.format(wikiget_version, mwclient_version)) +DEFAULT_SITE = "commons.wikimedia.org" +DEFAULT_PATH = "/w/" +USER_AGENT = (f"wikiget/{wikiget_version} (https://github.com/clpo13/wikiget) " + f"mwclient/{mwclient_version}") diff --git a/wikiget/dl.py b/wikiget/dl.py index 856d8ca..8f32218 100644 --- a/wikiget/dl.py +++ b/wikiget/dl.py @@ -102,11 +102,12 @@ def download(dl, args): if file.imageinfo != {}: # file exists either locally or at a common repository, # like Wikimedia Commons - file_url = file.imageinfo['url'] - file_size = file.imageinfo['size'] - file_sha1 = file.imageinfo['sha1'] + file_url = file.imageinfo["url"] + file_size = file.imageinfo["size"] + file_sha1 = file.imageinfo["sha1"] - filename_log = f"Downloading '{filename}' ({file_size} bytes) from {site.host}" + filename_log = (f"Downloading '{filename}' ({file_size} bytes) " + f"from {site.host}") if args.output: filename_log += f" to '{dest}'" logging.info(filename_log) @@ -117,7 +118,7 @@ def download(dl, args): "(use -f to ignore)") else: try: - fd = open(dest, 'wb') + fd = open(dest, "wb") except IOError as e: logging.error("File could not be written. " "The following error was encountered:") @@ -130,7 +131,7 @@ def download(dl, args): else: leave_bars = False with tqdm(leave=leave_bars, total=file_size, - unit='B', unit_scale=True, + unit="B", unit_scale=True, unit_divisor=CHUNKSIZE) as progress_bar: with fd: res = site.connection.get(file_url, stream=True) diff --git a/wikiget/validations.py b/wikiget/validations.py index 20ef74f..bd99570 100644 --- a/wikiget/validations.py +++ b/wikiget/validations.py @@ -31,7 +31,7 @@ def valid_file(search_string): """ # second group could also restrict to file extensions with three or more # letters with ([^/\r\n\t\f\v]+\.\w{3,}) - file_regex = re.compile(r'(File:|Image:)([^/\r\n\t\f\v]+\.\w+)$', re.I) + file_regex = re.compile(r"(File:|Image:)([^/\r\n\t\f\v]+\.\w+)$", re.I) return file_regex.search(search_string) @@ -44,7 +44,7 @@ def valid_site(search_string): :param search_string: string to validate :returns: a regex Match object if there's a match or None otherwise """ - site_regex = re.compile(r'wiki[mp]edia\.org$', re.I) + site_regex = re.compile(r"wiki[mp]edia\.org$", re.I) return site_regex.search(search_string) @@ -56,7 +56,7 @@ def verify_hash(filename): :return: hash digest """ hasher = hashlib.sha1() - with open(filename, 'rb') as dl: + with open(filename, "rb") as dl: buf = dl.read(BLOCKSIZE) while len(buf) > 0: hasher.update(buf) diff --git a/wikiget/version.py b/wikiget/version.py index 93b60a1..dd9b22c 100644 --- a/wikiget/version.py +++ b/wikiget/version.py @@ -1 +1 @@ -__version__ = '0.5.1' +__version__ = "0.5.1" diff --git a/wikiget/wikiget.py b/wikiget/wikiget.py index 6a537ba..a8679c9 100644 --- a/wikiget/wikiget.py +++ b/wikiget/wikiget.py @@ -44,42 +44,42 @@ def main(): conditions. There is NO WARRANTY, to the extent permitted by law. """) - parser.add_argument('FILE', help=""" + parser.add_argument("FILE", help=""" name of the file to download with the File: prefix, or the URL of its file description page """) - parser.add_argument('-V', '--version', action='version', - version=f'%(prog)s {wikiget_version}') + parser.add_argument("-V", "--version", action="version", + version=f"%(prog)s {wikiget_version}") message_options = parser.add_mutually_exclusive_group() - message_options.add_argument('-q', '--quiet', - help='suppress warning messages', - action='store_true') - message_options.add_argument('-v', '--verbose', - help='print detailed information; ' - 'use -vv for even more detail', - action='count', default=0) - parser.add_argument('-f', '--force', - help='force overwriting existing files', - action='store_true') - parser.add_argument('-s', '--site', default=DEFAULT_SITE, - help='MediaWiki site to download from ' - '(default: %(default)s)') - parser.add_argument('-p', '--path', default=DEFAULT_PATH, - help='MediaWiki site path, where api.php is located ' - '(default: %(default)s)') - parser.add_argument('--username', default='', - help='MediaWiki site username, for private wikis') - parser.add_argument('--password', default='', - help='MediaWiki site password, for private wikis') + message_options.add_argument("-q", "--quiet", + help="suppress warning messages", + action="store_true") + message_options.add_argument("-v", "--verbose", + help="print detailed information; " + "use -vv for even more detail", + action="count", default=0) + parser.add_argument("-f", "--force", + help="force overwriting existing files", + action="store_true") + parser.add_argument("-s", "--site", default=DEFAULT_SITE, + help="MediaWiki site to download from " + "(default: %(default)s)") + parser.add_argument("-p", "--path", default=DEFAULT_PATH, + help="MediaWiki site path, where api.php is located " + "(default: %(default)s)") + parser.add_argument("--username", default="", + help="MediaWiki site username, for private wikis") + parser.add_argument("--password", default="", + help="MediaWiki site password, for private wikis") output_options = parser.add_mutually_exclusive_group() - output_options.add_argument('-o', '--output', - help='write download to OUTPUT') - output_options.add_argument('-a', '--batch', - help='treat FILE as a textfile containing ' - 'multiple files to download, one URL or ' - 'filename per line', action='store_true') - parser.add_argument('-l', '--logfile', default='', - help='save log output to LOGFILE') + output_options.add_argument("-o", "--output", + help="write download to OUTPUT") + output_options.add_argument("-a", "--batch", + help="treat FILE as a textfile containing " + "multiple files to download, one URL or " + "filename per line", action="store_true") + parser.add_argument("-l", "--logfile", default="", + help="save log output to LOGFILE") args = parser.parse_args() @@ -131,7 +131,7 @@ def main(): logging.info(f"Using batch file '{input_file}'.") try: - fd = open(input_file, 'r') + fd = open(input_file, "r") except IOError as e: logging.error("File could not be read. " "The following error was encountered:") -- cgit v1.2.3 From a1995912ed24b37a990f3fcd5e91dbf7b46669fb Mon Sep 17 00:00:00 2001 From: Cody Logan Date: Tue, 26 Sep 2023 15:17:04 -0700 Subject: Reorganize file tree --- src/wikiget/__init__.py | 28 ++++++++ src/wikiget/dl.py | 159 +++++++++++++++++++++++++++++++++++++++++++++ src/wikiget/validations.py | 64 ++++++++++++++++++ src/wikiget/version.py | 1 + src/wikiget/wikiget.py | 157 ++++++++++++++++++++++++++++++++++++++++++++ test/test_validations.py | 91 -------------------------- tests/test_validations.py | 91 ++++++++++++++++++++++++++ wikiget/__init__.py | 28 -------- wikiget/dl.py | 159 --------------------------------------------- wikiget/validations.py | 64 ------------------ wikiget/version.py | 1 - wikiget/wikiget.py | 157 -------------------------------------------- 12 files changed, 500 insertions(+), 500 deletions(-) create mode 100644 src/wikiget/__init__.py create mode 100644 src/wikiget/dl.py create mode 100644 src/wikiget/validations.py create mode 100644 src/wikiget/version.py create mode 100644 src/wikiget/wikiget.py delete mode 100644 test/test_validations.py create mode 100644 tests/test_validations.py delete mode 100644 wikiget/__init__.py delete mode 100644 wikiget/dl.py delete mode 100644 wikiget/validations.py delete mode 100644 wikiget/version.py delete mode 100644 wikiget/wikiget.py diff --git a/src/wikiget/__init__.py b/src/wikiget/__init__.py new file mode 100644 index 0000000..4adcae3 --- /dev/null +++ b/src/wikiget/__init__.py @@ -0,0 +1,28 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2018-2021 Cody Logan and contributors +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see . + +from mwclient import __version__ as mwclient_version + +from .version import __version__ as wikiget_version + +# set some global constants +BLOCKSIZE = 65536 +CHUNKSIZE = 1024 +DEFAULT_SITE = "commons.wikimedia.org" +DEFAULT_PATH = "/w/" +USER_AGENT = (f"wikiget/{wikiget_version} (https://github.com/clpo13/wikiget) " + f"mwclient/{mwclient_version}") diff --git a/src/wikiget/dl.py b/src/wikiget/dl.py new file mode 100644 index 0000000..8f32218 --- /dev/null +++ b/src/wikiget/dl.py @@ -0,0 +1,159 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2018-2021 Cody Logan and contributors +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see . + +import logging +import os +import sys +from urllib.parse import unquote, urlparse + +from mwclient import APIError, InvalidResponse, LoginError, Site +from requests import ConnectionError, HTTPError +from tqdm import tqdm + +from . import CHUNKSIZE, DEFAULT_SITE, USER_AGENT +from .validations import valid_file, verify_hash + + +def download(dl, args): + url = urlparse(dl) + + if url.netloc: + filename = url.path + site_name = url.netloc + if args.site is not DEFAULT_SITE: + # this will work even if the user specifies 'commons.wikimedia.org' + logging.warning("target is a URL, " + "ignoring site specified with --site") + else: + filename = dl + site_name = args.site + + file_match = valid_file(filename) + + # check if this is a valid file + if file_match and file_match.group(1): + # has File:/Image: prefix and extension + filename = file_match.group(2) + else: + # no file extension and/or prefix, probably an article + logging.error(f"Could not parse input '{filename}' as a file.") + sys.exit(1) + + filename = unquote(filename) # remove URL encoding for special characters + + dest = args.output or filename + + logging.debug(f"User agent: {USER_AGENT}") + + # connect to site and identify ourselves + logging.info(f"Site name: {site_name}") + try: + site = Site(site_name, path=args.path, clients_useragent=USER_AGENT) + if args.username and args.password: + site.login(args.username, args.password) + except ConnectionError as e: + # usually this means there is no such site, or there's no network + # connection, though it could be a certificate problem + logging.error("Couldn't connect to specified site.") + logging.debug("Full error message:") + logging.debug(e) + sys.exit(1) + except HTTPError as e: + # most likely a 403 forbidden or 404 not found error for api.php + logging.error("Couldn't find the specified wiki's api.php. " + "Check the value of --path.") + logging.debug("Full error message:") + logging.debug(e) + sys.exit(1) + except (InvalidResponse, LoginError) as e: + # InvalidResponse: site exists, but we couldn't communicate with the + # API endpoint for some reason other than an HTTP error. + # LoginError: missing or invalid credentials + logging.error(e) + sys.exit(1) + + # get info about the target file + try: + file = site.images[filename] + except APIError as e: + # an API error at this point likely means access is denied, + # which could happen with a private wiki + logging.error("Access denied. Try providing credentials with " + "--username and --password.") + logging.debug("Full error message:") + for i in e.args: + logging.debug(i) + sys.exit(1) + + if file.imageinfo != {}: + # file exists either locally or at a common repository, + # like Wikimedia Commons + file_url = file.imageinfo["url"] + file_size = file.imageinfo["size"] + file_sha1 = file.imageinfo["sha1"] + + filename_log = (f"Downloading '{filename}' ({file_size} bytes) " + f"from {site.host}") + if args.output: + filename_log += f" to '{dest}'" + logging.info(filename_log) + logging.info(f"{file_url}") + + if os.path.isfile(dest) and not args.force: + logging.warning(f"File '{dest}' already exists, skipping download " + "(use -f to ignore)") + else: + try: + fd = open(dest, "wb") + except IOError as e: + logging.error("File could not be written. " + "The following error was encountered:") + logging.error(e) + sys.exit(1) + else: + # download the file(s) + if args.verbose >= 1: + leave_bars = True + else: + leave_bars = False + with tqdm(leave=leave_bars, total=file_size, + unit="B", unit_scale=True, + unit_divisor=CHUNKSIZE) as progress_bar: + with fd: + res = site.connection.get(file_url, stream=True) + progress_bar.set_postfix(file=dest, refresh=False) + for chunk in res.iter_content(CHUNKSIZE): + fd.write(chunk) + progress_bar.update(len(chunk)) + + # verify file integrity and optionally print details + dl_sha1 = verify_hash(dest) + + logging.info(f"Downloaded file SHA1 is {dl_sha1}") + logging.info(f"Server file SHA1 is {file_sha1}") + if dl_sha1 == file_sha1: + logging.info("Hashes match!") + # at this point, we've successfully downloaded the file + else: + logging.error("Hash mismatch! Downloaded file may be corrupt.") + sys.exit(1) + + else: + # no file information returned + logging.error(f"Target '{filename}' does not appear to be " + "a valid file.") + sys.exit(1) diff --git a/src/wikiget/validations.py b/src/wikiget/validations.py new file mode 100644 index 0000000..bd99570 --- /dev/null +++ b/src/wikiget/validations.py @@ -0,0 +1,64 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2018, 2019, 2020 Cody Logan +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see . + +import hashlib +import re + +from . import BLOCKSIZE + + +def valid_file(search_string): + """ + Determines if the given string contains a valid file name, defined as a + string ending with a '.' and at least one character, beginning with 'File:' + or 'Image:', the standard file prefixes in MediaWiki. + :param search_string: string to validate + :returns: a regex Match object if there's a match or None otherwise + """ + # second group could also restrict to file extensions with three or more + # letters with ([^/\r\n\t\f\v]+\.\w{3,}) + file_regex = re.compile(r"(File:|Image:)([^/\r\n\t\f\v]+\.\w+)$", re.I) + return file_regex.search(search_string) + + +def valid_site(search_string): + """ + Determines if the given string contains a valid site name, defined as a + string ending with 'wikipedia.org' or 'wikimedia.org'. This covers all + subdomains of those domains. Eventually, it should be possible to support + any MediaWiki site, regardless of domain name. + :param search_string: string to validate + :returns: a regex Match object if there's a match or None otherwise + """ + site_regex = re.compile(r"wiki[mp]edia\.org$", re.I) + return site_regex.search(search_string) + + +def verify_hash(filename): + """ + Calculates the SHA1 hash of the given file for comparison with a known + value. + :param filename: name of the file to calculate a hash for + :return: hash digest + """ + hasher = hashlib.sha1() + with open(filename, "rb") as dl: + buf = dl.read(BLOCKSIZE) + while len(buf) > 0: + hasher.update(buf) + buf = dl.read(BLOCKSIZE) + return hasher.hexdigest() diff --git a/src/wikiget/version.py b/src/wikiget/version.py new file mode 100644 index 0000000..dd9b22c --- /dev/null +++ b/src/wikiget/version.py @@ -0,0 +1 @@ +__version__ = "0.5.1" diff --git a/src/wikiget/wikiget.py b/src/wikiget/wikiget.py new file mode 100644 index 0000000..a8679c9 --- /dev/null +++ b/src/wikiget/wikiget.py @@ -0,0 +1,157 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2018-2021 Cody Logan and contributors +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see . + +import argparse +import logging +import sys + +from . import DEFAULT_SITE, DEFAULT_PATH, wikiget_version +from .dl import download + + +def main(): + """ + Main entry point for console script. Automatically compiled by setuptools + when installed with `pip install` or `python setup.py install`. + """ + + parser = argparse.ArgumentParser(description=""" + A tool for downloading files from + MediaWiki sites using the file name or + description page URL + """, + epilog=""" + Copyright (C) 2018-2021 Cody Logan + and contributors. + License GPLv3+: GNU GPL version 3 or later + . + This is free software; you are free to + change and redistribute it under certain + conditions. There is NO WARRANTY, to the + extent permitted by law. + """) + parser.add_argument("FILE", help=""" + name of the file to download with the File: + prefix, or the URL of its file description page + """) + parser.add_argument("-V", "--version", action="version", + version=f"%(prog)s {wikiget_version}") + message_options = parser.add_mutually_exclusive_group() + message_options.add_argument("-q", "--quiet", + help="suppress warning messages", + action="store_true") + message_options.add_argument("-v", "--verbose", + help="print detailed information; " + "use -vv for even more detail", + action="count", default=0) + parser.add_argument("-f", "--force", + help="force overwriting existing files", + action="store_true") + parser.add_argument("-s", "--site", default=DEFAULT_SITE, + help="MediaWiki site to download from " + "(default: %(default)s)") + parser.add_argument("-p", "--path", default=DEFAULT_PATH, + help="MediaWiki site path, where api.php is located " + "(default: %(default)s)") + parser.add_argument("--username", default="", + help="MediaWiki site username, for private wikis") + parser.add_argument("--password", default="", + help="MediaWiki site password, for private wikis") + output_options = parser.add_mutually_exclusive_group() + output_options.add_argument("-o", "--output", + help="write download to OUTPUT") + output_options.add_argument("-a", "--batch", + help="treat FILE as a textfile containing " + "multiple files to download, one URL or " + "filename per line", action="store_true") + parser.add_argument("-l", "--logfile", default="", + help="save log output to LOGFILE") + + args = parser.parse_args() + + loglevel = logging.WARNING + if args.verbose >= 2: + # this includes API and library messages + loglevel = logging.DEBUG + elif args.verbose >= 1: + loglevel = logging.INFO + elif args.quiet: + loglevel = logging.ERROR + + # configure logging: + # console log level is set via -v, -vv, and -q options + # file log level is always info (TODO: add debug option) + if args.logfile: + # log to console and file + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)-7s] %(message)s", + filename=args.logfile + ) + + console = logging.StreamHandler() + # TODO: even when loglevel is set to logging.DEBUG, + # debug messages aren't printing to console + console.setLevel(loglevel) + console.setFormatter( + logging.Formatter("[%(levelname)s] %(message)s") + ) + logging.getLogger("").addHandler(console) + else: + # log only to console + logging.basicConfig( + level=loglevel, + format="[%(levelname)s] %(message)s" + ) + + # log events are appended to the file if it already exists, + # so note the start of a new download session + logging.info(f"Starting download session using wikiget {wikiget_version}") + # logging.info(f"Log level is set to {loglevel}") + + if args.batch: + # batch download mode + input_file = args.FILE + dl_list = [] + + logging.info(f"Using batch file '{input_file}'.") + + try: + fd = open(input_file, "r") + except IOError as e: + logging.error("File could not be read. " + "The following error was encountered:") + logging.error(e) + sys.exit(1) + else: + with fd: + # store file contents in memory in case something + # happens to the file while we're downloading + for _, line in enumerate(fd): + dl_list.append(line) + + # TODO: validate file contents before download process starts + for line_num, url in enumerate(dl_list, start=1): + url = url.strip() + # keep track of batch file line numbers for + # debugging/logging purposes + logging.info(f"Downloading '{url}' at line {line_num}:") + download(url, args) + else: + # single download mode + dl = args.FILE + download(dl, args) diff --git a/test/test_validations.py b/test/test_validations.py deleted file mode 100644 index 8dd4d6d..0000000 --- a/test/test_validations.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -# wikiget - CLI tool for downloading files from Wikimedia sites -# Copyright (C) 2018-2021 Cody Logan -# SPDX-License-Identifier: GPL-3.0-or-later -# -# Wikiget is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Wikiget is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Wikiget. If not, see . - -from wikiget.validations import valid_file, valid_site, verify_hash - - -def test_invalid_site_input(): - """ - Invalid site strings should not return regex match objects. - """ - invalid_input = ["example.com", "vim.wikia.com", - "en.wikipedia.com", "en.wikimpedia.org"] - for i in invalid_input: - site_match = valid_site(i) - assert site_match is None - - -def test_valid_site_input(): - """ - Valid site strings should return regex match objects. - """ - valid_input = ["en.wikipedia.org", "commons.wikimedia.org", - "de.wikipedia.org", "meta.wikimedia.org"] - for i in valid_input: - site_match = valid_site(i) - assert site_match is not None - - -def test_file_regex(): - """ - File regex should return a match object with match groups corresponding - to the file prefix and name. - """ - i = "File:Example.jpg" - file_match = valid_file(i) - assert file_match is not None - assert file_match.group(0) == "File:Example.jpg" # entire match - assert file_match.group(1) == "File:" # first group - assert file_match.group(2) == "Example.jpg" # second group - - -def test_invalid_file_input(): - """ - Invalid file strings should not return regex match objects. - """ - invalid_input = ["file:example", "example.jpg", "Foo Bar.gif", - "Fil:Example.jpg"] - for i in invalid_input: - file_match = valid_file(i) - assert file_match is None - - -def test_valid_file_input(): - """ - Valid file strings should return regex match objects. - """ - valid_input = ["Image:example.jpg", "file:example.jpg", - "File:example.file-01.jpg", "FILE:FOO.BMP", - "File:ß handwritten sample.gif", "File:A (1).jpeg"] - for i in valid_input: - file_match = valid_file(i) - assert file_match is not None - - -def test_verify_hash(tmp_path): - """ - Confirm that verify_hash returns the proper SHA1 hash. - """ - file_name = "testfile" - file_contents = "foobar" - file_sha1 = "8843d7f92416211de9ebb963ff4ce28125932878" - - tmp_file = tmp_path / file_name - tmp_file.write_text(file_contents) - - assert verify_hash(tmp_file) == file_sha1 diff --git a/tests/test_validations.py b/tests/test_validations.py new file mode 100644 index 0000000..8dd4d6d --- /dev/null +++ b/tests/test_validations.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2018-2021 Cody Logan +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see . + +from wikiget.validations import valid_file, valid_site, verify_hash + + +def test_invalid_site_input(): + """ + Invalid site strings should not return regex match objects. + """ + invalid_input = ["example.com", "vim.wikia.com", + "en.wikipedia.com", "en.wikimpedia.org"] + for i in invalid_input: + site_match = valid_site(i) + assert site_match is None + + +def test_valid_site_input(): + """ + Valid site strings should return regex match objects. + """ + valid_input = ["en.wikipedia.org", "commons.wikimedia.org", + "de.wikipedia.org", "meta.wikimedia.org"] + for i in valid_input: + site_match = valid_site(i) + assert site_match is not None + + +def test_file_regex(): + """ + File regex should return a match object with match groups corresponding + to the file prefix and name. + """ + i = "File:Example.jpg" + file_match = valid_file(i) + assert file_match is not None + assert file_match.group(0) == "File:Example.jpg" # entire match + assert file_match.group(1) == "File:" # first group + assert file_match.group(2) == "Example.jpg" # second group + + +def test_invalid_file_input(): + """ + Invalid file strings should not return regex match objects. + """ + invalid_input = ["file:example", "example.jpg", "Foo Bar.gif", + "Fil:Example.jpg"] + for i in invalid_input: + file_match = valid_file(i) + assert file_match is None + + +def test_valid_file_input(): + """ + Valid file strings should return regex match objects. + """ + valid_input = ["Image:example.jpg", "file:example.jpg", + "File:example.file-01.jpg", "FILE:FOO.BMP", + "File:ß handwritten sample.gif", "File:A (1).jpeg"] + for i in valid_input: + file_match = valid_file(i) + assert file_match is not None + + +def test_verify_hash(tmp_path): + """ + Confirm that verify_hash returns the proper SHA1 hash. + """ + file_name = "testfile" + file_contents = "foobar" + file_sha1 = "8843d7f92416211de9ebb963ff4ce28125932878" + + tmp_file = tmp_path / file_name + tmp_file.write_text(file_contents) + + assert verify_hash(tmp_file) == file_sha1 diff --git a/wikiget/__init__.py b/wikiget/__init__.py deleted file mode 100644 index 4adcae3..0000000 --- a/wikiget/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# wikiget - CLI tool for downloading files from Wikimedia sites -# Copyright (C) 2018-2021 Cody Logan and contributors -# SPDX-License-Identifier: GPL-3.0-or-later -# -# Wikiget is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Wikiget is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Wikiget. If not, see . - -from mwclient import __version__ as mwclient_version - -from .version import __version__ as wikiget_version - -# set some global constants -BLOCKSIZE = 65536 -CHUNKSIZE = 1024 -DEFAULT_SITE = "commons.wikimedia.org" -DEFAULT_PATH = "/w/" -USER_AGENT = (f"wikiget/{wikiget_version} (https://github.com/clpo13/wikiget) " - f"mwclient/{mwclient_version}") diff --git a/wikiget/dl.py b/wikiget/dl.py deleted file mode 100644 index 8f32218..0000000 --- a/wikiget/dl.py +++ /dev/null @@ -1,159 +0,0 @@ -# wikiget - CLI tool for downloading files from Wikimedia sites -# Copyright (C) 2018-2021 Cody Logan and contributors -# SPDX-License-Identifier: GPL-3.0-or-later -# -# Wikiget is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Wikiget is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Wikiget. If not, see . - -import logging -import os -import sys -from urllib.parse import unquote, urlparse - -from mwclient import APIError, InvalidResponse, LoginError, Site -from requests import ConnectionError, HTTPError -from tqdm import tqdm - -from . import CHUNKSIZE, DEFAULT_SITE, USER_AGENT -from .validations import valid_file, verify_hash - - -def download(dl, args): - url = urlparse(dl) - - if url.netloc: - filename = url.path - site_name = url.netloc - if args.site is not DEFAULT_SITE: - # this will work even if the user specifies 'commons.wikimedia.org' - logging.warning("target is a URL, " - "ignoring site specified with --site") - else: - filename = dl - site_name = args.site - - file_match = valid_file(filename) - - # check if this is a valid file - if file_match and file_match.group(1): - # has File:/Image: prefix and extension - filename = file_match.group(2) - else: - # no file extension and/or prefix, probably an article - logging.error(f"Could not parse input '{filename}' as a file.") - sys.exit(1) - - filename = unquote(filename) # remove URL encoding for special characters - - dest = args.output or filename - - logging.debug(f"User agent: {USER_AGENT}") - - # connect to site and identify ourselves - logging.info(f"Site name: {site_name}") - try: - site = Site(site_name, path=args.path, clients_useragent=USER_AGENT) - if args.username and args.password: - site.login(args.username, args.password) - except ConnectionError as e: - # usually this means there is no such site, or there's no network - # connection, though it could be a certificate problem - logging.error("Couldn't connect to specified site.") - logging.debug("Full error message:") - logging.debug(e) - sys.exit(1) - except HTTPError as e: - # most likely a 403 forbidden or 404 not found error for api.php - logging.error("Couldn't find the specified wiki's api.php. " - "Check the value of --path.") - logging.debug("Full error message:") - logging.debug(e) - sys.exit(1) - except (InvalidResponse, LoginError) as e: - # InvalidResponse: site exists, but we couldn't communicate with the - # API endpoint for some reason other than an HTTP error. - # LoginError: missing or invalid credentials - logging.error(e) - sys.exit(1) - - # get info about the target file - try: - file = site.images[filename] - except APIError as e: - # an API error at this point likely means access is denied, - # which could happen with a private wiki - logging.error("Access denied. Try providing credentials with " - "--username and --password.") - logging.debug("Full error message:") - for i in e.args: - logging.debug(i) - sys.exit(1) - - if file.imageinfo != {}: - # file exists either locally or at a common repository, - # like Wikimedia Commons - file_url = file.imageinfo["url"] - file_size = file.imageinfo["size"] - file_sha1 = file.imageinfo["sha1"] - - filename_log = (f"Downloading '{filename}' ({file_size} bytes) " - f"from {site.host}") - if args.output: - filename_log += f" to '{dest}'" - logging.info(filename_log) - logging.info(f"{file_url}") - - if os.path.isfile(dest) and not args.force: - logging.warning(f"File '{dest}' already exists, skipping download " - "(use -f to ignore)") - else: - try: - fd = open(dest, "wb") - except IOError as e: - logging.error("File could not be written. " - "The following error was encountered:") - logging.error(e) - sys.exit(1) - else: - # download the file(s) - if args.verbose >= 1: - leave_bars = True - else: - leave_bars = False - with tqdm(leave=leave_bars, total=file_size, - unit="B", unit_scale=True, - unit_divisor=CHUNKSIZE) as progress_bar: - with fd: - res = site.connection.get(file_url, stream=True) - progress_bar.set_postfix(file=dest, refresh=False) - for chunk in res.iter_content(CHUNKSIZE): - fd.write(chunk) - progress_bar.update(len(chunk)) - - # verify file integrity and optionally print details - dl_sha1 = verify_hash(dest) - - logging.info(f"Downloaded file SHA1 is {dl_sha1}") - logging.info(f"Server file SHA1 is {file_sha1}") - if dl_sha1 == file_sha1: - logging.info("Hashes match!") - # at this point, we've successfully downloaded the file - else: - logging.error("Hash mismatch! Downloaded file may be corrupt.") - sys.exit(1) - - else: - # no file information returned - logging.error(f"Target '{filename}' does not appear to be " - "a valid file.") - sys.exit(1) diff --git a/wikiget/validations.py b/wikiget/validations.py deleted file mode 100644 index bd99570..0000000 --- a/wikiget/validations.py +++ /dev/null @@ -1,64 +0,0 @@ -# wikiget - CLI tool for downloading files from Wikimedia sites -# Copyright (C) 2018, 2019, 2020 Cody Logan -# SPDX-License-Identifier: GPL-3.0-or-later -# -# Wikiget is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Wikiget is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Wikiget. If not, see . - -import hashlib -import re - -from . import BLOCKSIZE - - -def valid_file(search_string): - """ - Determines if the given string contains a valid file name, defined as a - string ending with a '.' and at least one character, beginning with 'File:' - or 'Image:', the standard file prefixes in MediaWiki. - :param search_string: string to validate - :returns: a regex Match object if there's a match or None otherwise - """ - # second group could also restrict to file extensions with three or more - # letters with ([^/\r\n\t\f\v]+\.\w{3,}) - file_regex = re.compile(r"(File:|Image:)([^/\r\n\t\f\v]+\.\w+)$", re.I) - return file_regex.search(search_string) - - -def valid_site(search_string): - """ - Determines if the given string contains a valid site name, defined as a - string ending with 'wikipedia.org' or 'wikimedia.org'. This covers all - subdomains of those domains. Eventually, it should be possible to support - any MediaWiki site, regardless of domain name. - :param search_string: string to validate - :returns: a regex Match object if there's a match or None otherwise - """ - site_regex = re.compile(r"wiki[mp]edia\.org$", re.I) - return site_regex.search(search_string) - - -def verify_hash(filename): - """ - Calculates the SHA1 hash of the given file for comparison with a known - value. - :param filename: name of the file to calculate a hash for - :return: hash digest - """ - hasher = hashlib.sha1() - with open(filename, "rb") as dl: - buf = dl.read(BLOCKSIZE) - while len(buf) > 0: - hasher.update(buf) - buf = dl.read(BLOCKSIZE) - return hasher.hexdigest() diff --git a/wikiget/version.py b/wikiget/version.py deleted file mode 100644 index dd9b22c..0000000 --- a/wikiget/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.5.1" diff --git a/wikiget/wikiget.py b/wikiget/wikiget.py deleted file mode 100644 index a8679c9..0000000 --- a/wikiget/wikiget.py +++ /dev/null @@ -1,157 +0,0 @@ -# wikiget - CLI tool for downloading files from Wikimedia sites -# Copyright (C) 2018-2021 Cody Logan and contributors -# SPDX-License-Identifier: GPL-3.0-or-later -# -# Wikiget is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Wikiget is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Wikiget. If not, see . - -import argparse -import logging -import sys - -from . import DEFAULT_SITE, DEFAULT_PATH, wikiget_version -from .dl import download - - -def main(): - """ - Main entry point for console script. Automatically compiled by setuptools - when installed with `pip install` or `python setup.py install`. - """ - - parser = argparse.ArgumentParser(description=""" - A tool for downloading files from - MediaWiki sites using the file name or - description page URL - """, - epilog=""" - Copyright (C) 2018-2021 Cody Logan - and contributors. - License GPLv3+: GNU GPL version 3 or later - . - This is free software; you are free to - change and redistribute it under certain - conditions. There is NO WARRANTY, to the - extent permitted by law. - """) - parser.add_argument("FILE", help=""" - name of the file to download with the File: - prefix, or the URL of its file description page - """) - parser.add_argument("-V", "--version", action="version", - version=f"%(prog)s {wikiget_version}") - message_options = parser.add_mutually_exclusive_group() - message_options.add_argument("-q", "--quiet", - help="suppress warning messages", - action="store_true") - message_options.add_argument("-v", "--verbose", - help="print detailed information; " - "use -vv for even more detail", - action="count", default=0) - parser.add_argument("-f", "--force", - help="force overwriting existing files", - action="store_true") - parser.add_argument("-s", "--site", default=DEFAULT_SITE, - help="MediaWiki site to download from " - "(default: %(default)s)") - parser.add_argument("-p", "--path", default=DEFAULT_PATH, - help="MediaWiki site path, where api.php is located " - "(default: %(default)s)") - parser.add_argument("--username", default="", - help="MediaWiki site username, for private wikis") - parser.add_argument("--password", default="", - help="MediaWiki site password, for private wikis") - output_options = parser.add_mutually_exclusive_group() - output_options.add_argument("-o", "--output", - help="write download to OUTPUT") - output_options.add_argument("-a", "--batch", - help="treat FILE as a textfile containing " - "multiple files to download, one URL or " - "filename per line", action="store_true") - parser.add_argument("-l", "--logfile", default="", - help="save log output to LOGFILE") - - args = parser.parse_args() - - loglevel = logging.WARNING - if args.verbose >= 2: - # this includes API and library messages - loglevel = logging.DEBUG - elif args.verbose >= 1: - loglevel = logging.INFO - elif args.quiet: - loglevel = logging.ERROR - - # configure logging: - # console log level is set via -v, -vv, and -q options - # file log level is always info (TODO: add debug option) - if args.logfile: - # log to console and file - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)-7s] %(message)s", - filename=args.logfile - ) - - console = logging.StreamHandler() - # TODO: even when loglevel is set to logging.DEBUG, - # debug messages aren't printing to console - console.setLevel(loglevel) - console.setFormatter( - logging.Formatter("[%(levelname)s] %(message)s") - ) - logging.getLogger("").addHandler(console) - else: - # log only to console - logging.basicConfig( - level=loglevel, - format="[%(levelname)s] %(message)s" - ) - - # log events are appended to the file if it already exists, - # so note the start of a new download session - logging.info(f"Starting download session using wikiget {wikiget_version}") - # logging.info(f"Log level is set to {loglevel}") - - if args.batch: - # batch download mode - input_file = args.FILE - dl_list = [] - - logging.info(f"Using batch file '{input_file}'.") - - try: - fd = open(input_file, "r") - except IOError as e: - logging.error("File could not be read. " - "The following error was encountered:") - logging.error(e) - sys.exit(1) - else: - with fd: - # store file contents in memory in case something - # happens to the file while we're downloading - for _, line in enumerate(fd): - dl_list.append(line) - - # TODO: validate file contents before download process starts - for line_num, url in enumerate(dl_list, start=1): - url = url.strip() - # keep track of batch file line numbers for - # debugging/logging purposes - logging.info(f"Downloading '{url}' at line {line_num}:") - download(url, args) - else: - # single download mode - dl = args.FILE - download(dl, args) -- cgit v1.2.3