aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCody Logan <clpo13@gmail.com>2020-01-03 11:03:12 -0800
committerCody Logan <clpo13@gmail.com>2020-01-03 11:03:12 -0800
commit3c2d03f02e644b8afc5178f7c1a11d50d6b21118 (patch)
tree6180b8e6a377db05785f063c0795e9c079ea55a8
parent8273f4cdc3a4ee67d936c2b0b06f3d5ee92c31bf (diff)
downloadwikiget-3c2d03f02e644b8afc5178f7c1a11d50d6b21118.tar.gz
wikiget-3c2d03f02e644b8afc5178f7c1a11d50d6b21118.zip
Drop support for Python 2.
-rw-r--r--.travis.yml3
-rw-r--r--README.md4
-rw-r--r--setup.py56
-rw-r--r--wikiget/version.py2
-rw-r--r--wikiget/wikiget.py151
5 files changed, 114 insertions, 102 deletions
diff --git a/.travis.yml b/.travis.yml
index cda16db..d373bba 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,6 @@
language: python
dist: xenial
python:
-- '2.7'
- '3.5'
- '3.6'
- '3.7'
@@ -17,5 +16,5 @@ deploy:
secure: KvcviHqqT4YprtmzAtf9w8BkKPfwJ53LIOXmUFotzT1Qjt3FSE7bWVzDItFjy54zZM+tqKAniL91R+2tM5uQFn4fVS/yykN1Akts6ZnkJdq99Lgdb1V3gEv366K5AWoYKgjZX+PRvmOk8BXSrqbVtXN0lhmoemmeJVDqDHg2HJZNYFwvmr/g64amm2d/cdfLxKHpduwciNY6xUhOFIdlbrJ1T767mpC+gnqfzmJeNF7K95pmyBF6Wvl4AkKzwJJkyZULQF2VFtIT6bzSuM6G26ZT6H7UyoP+8+CvI4Fe6h8Ol7sWSuVC5gz+5istRORUy8RQ22HWW1ZZKOw1+8/dHuBPvIZOnfcTvNw07e7267KUoO4FfGLvTxU2likorr5gZh1YaCNut6XJkjzwNddkutCXv65H7zOhSn2gl7vMFkUUf+kEM9pSBcA1zf7Y9+7U3HgyD1OH+a5jRIOe0Vy9r3PPaMXuDgsHxZkrVlsr3LgtGwFD0jWMDZtROXds6OXW6/n6cN30IPSf/qWdgduNIq3wj0JbALI5AB0rugNNPhePMVOfF90W9WLPFlxQCjLji8NpvM5341bS8aLhFIgIfRGDgG9AN+I/dZNIwD2J0vfw/BDaLVNc2XUAGLa359Lbz9bFYUp0J5B8hdMMyR6YULaN5alz2VsUC5sD6kPiqTo=
on:
tags: true
- distributions: sdist bdist_wheel --universal
+ distributions: sdist bdist_wheel
skip_existing: true
diff --git a/README.md b/README.md
index a6500ad..981f90a 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@ Something like wget for downloading a file from MediaWiki sites (like Wikipedia
or Wikimedia Commons) using only the file name or the URL of its description
page.
-Requires Python 2.7 or 3.5+. Install with `pip install --user wikiget` or, if you're using
+Requires Python 3.5+. Install with `pip install --user wikiget` or, if you're using
[Homebrew](https://brew.sh/), `brew tap clpo13/wikiget && brew install wikiget`.
## Usage
@@ -85,7 +85,7 @@ executable script. Unit tests can be run with `python setup.py test`.
## License
-Copyright (C) 2018, 2019 Cody Logan
+Copyright (C) 2018, 2019, 2020 Cody Logan
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
diff --git a/setup.py b/setup.py
index af030b3..89dc1cf 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,5 @@
# wikiget - CLI tool for downloading files from Wikimedia sites
-# Copyright (C) 2018, 2019 Cody Logan
+# Copyright (C) 2018, 2019, 2020 Cody Logan
# SPDX-License-Identifier: GPL-3.0-or-later
#
# Wikiget is free software: you can redistribute it and/or modify
@@ -23,45 +23,47 @@ from os import path
from setuptools import setup, find_packages
here = path.abspath(path.dirname(__file__))
-with open(path.join(here, "README.md"), "r") as fr:
+with open(path.join(here, 'README.md'), 'r') as fr:
long_description = fr.read()
version = {}
-with open(path.join(here, "wikiget", "version.py"), "r") as fv:
+with open(path.join(here, 'wikiget', 'version.py'), 'r') as fv:
exec(fv.read(), version)
setup(
- name="wikiget",
- version=version["__version__"],
- author="Cody Logan",
- author_email="clpo13@gmail.com",
- description="CLI tool for downloading files from MediaWiki sites",
+ name='wikiget',
+ version=version['__version__'],
+ author='Cody Logan',
+ author_email='clpo13@gmail.com',
+ description='CLI tool for downloading files from MediaWiki sites',
long_description=long_description,
- long_description_content_type="text/markdown",
- url="https://github.com/clpo13/wikiget",
- keywords="download mediawiki wikimedia wikipedia",
+ long_description_content_type='text/markdown',
+ url='https://github.com/clpo13/wikiget',
+ keywords='download mediawiki wikimedia wikipedia',
packages=find_packages(),
classifiers=[
- "Development Status :: 4 - Beta",
- "Environment :: Console",
- "Intended Audience :: End Users/Desktop",
- "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
- "Operating System :: OS Independent",
- "Programming Language :: Python :: 2.7",
- "Programming Language :: Python :: 3.5",
- "Programming Language :: Python :: 3.6",
- "Programming Language :: Python :: 3.7",
- "Programming Language :: Python :: 3.8",
- "Topic :: Utilities",
+ 'Development Status :: 4 - Beta',
+ 'Environment :: Console',
+ 'Intended Audience :: End Users/Desktop',
+ 'License :: OSI Approved :: GNU General Public License v3 or later '
+ '(GPLv3+)',
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.5',
+ 'Programming Language :: Python :: 3.6',
+ 'Programming Language :: Python :: 3.7',
+ 'Programming Language :: Python :: 3.8',
+ 'Topic :: Utilities',
],
- python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
- install_requires=["future", "mwclient>=0.10.0", "pytest-runner", "requests", "tqdm"],
- tests_require=["pytest"],
+ python_requires='>=3.5',
+ install_requires=['future', 'mwclient>=0.10.0', 'pytest-runner',
+ 'requests', 'tqdm'],
+ tests_require=['pytest'],
project_urls={
- "Bug Reports": "https://github.com/clpo13/wikiget/issues",
+ 'Bug Reports': 'https://github.com/clpo13/wikiget/issues',
},
entry_points={
- "console_scripts": [
+ 'console_scripts': [
'wikiget=wikiget.wikiget:main',
],
},
diff --git a/wikiget/version.py b/wikiget/version.py
index 0958641..d4bb133 100644
--- a/wikiget/version.py
+++ b/wikiget/version.py
@@ -1,3 +1,3 @@
"""Sets the program version in setup.py and on the command line."""
-__version__ = "0.2.1"
+__version__ = "0.3.0"
diff --git a/wikiget/wikiget.py b/wikiget/wikiget.py
index 8bcd3fd..a825c54 100644
--- a/wikiget/wikiget.py
+++ b/wikiget/wikiget.py
@@ -1,5 +1,5 @@
# wikiget - CLI tool for downloading files from Wikimedia sites
-# Copyright (C) 2018, 2019 Cody Logan
+# Copyright (C) 2018, 2019, 2020 Cody Logan
# SPDX-License-Identifier: GPL-3.0-or-later
#
# Wikiget is free software: you can redistribute it and/or modify
@@ -17,14 +17,6 @@
"""Main wikiget functions."""
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-from builtins import open
-
-from future import standard_library
-
-standard_library.install_aliases()
-
import argparse
import hashlib
import logging
@@ -40,9 +32,9 @@ from tqdm import tqdm
from wikiget.version import __version__
BLOCKSIZE = 65536
-DEFAULT_SITE = "en.wikipedia.org"
-USER_AGENT = "wikiget/{} (https://github.com/clpo13/wikiget) " \
- "mwclient/{}".format(__version__, mwclient_version)
+DEFAULT_SITE = 'en.wikipedia.org'
+USER_AGENT = 'wikiget/{} (https://github.com/clpo13/wikiget) ' \
+ 'mwclient/{}'.format(__version__, mwclient_version)
def main():
@@ -52,35 +44,43 @@ def main():
"""
parser = argparse.ArgumentParser(description="""
- A tool for downloading files from MediaWiki sites
- using the file name or description page URL
+ A tool for downloading files from
+ MediaWiki sites using the file name or
+ description page URL
""",
epilog="""
- Copyright (C) 2018, 2019 Cody Logan. License GPLv3+: GNU GPL
- version 3 or later <http://www.gnu.org/licenses/gpl.html>.
- This is free software; you are free to change and redistribute
- it under certain conditions. There is NO WARRANTY, to the
+ Copyright (C) 2018, 2019, 2020 Cody Logan.
+ License GPLv3+: GNU GPL version 3 or later
+ <http://www.gnu.org/licenses/gpl.html>.
+ This is free software; you are free to
+ change and redistribute it under certain
+ conditions. There is NO WARRANTY, to the
extent permitted by law.
""")
- parser.add_argument("FILE", help="""
- name of the file to download with the File: or Image: prefix,
- or the URL of its file description page
+ parser.add_argument('FILE', help="""
+ name of the file to download with the File: or Image:
+ prefix, or the URL of its file description page
""")
- parser.add_argument("-V", "--version", action="version",
- version="%(prog)s {}".format(__version__))
+ parser.add_argument('-V', '--version', action='version',
+ version='%(prog)s {}'.format(__version__))
output_options = parser.add_mutually_exclusive_group()
- output_options.add_argument("-q", "--quiet", help="suppress warning messages",
- action="store_true")
- output_options.add_argument("-v", "--verbose",
- help="print detailed information; use -vv for even more detail",
- action="count", default=0)
- parser.add_argument("-f", "--force", help="force overwriting existing files",
- action="store_true")
- parser.add_argument("-s", "--site", default=DEFAULT_SITE,
- help="MediaWiki site to download from (default: %(default)s)")
- parser.add_argument("-o", "--output", help="write download to OUTPUT")
- parser.add_argument("-a", "--batch", help="treat FILE as a textfile containing multiple files to download, one URL or filename per line",
- action="store_true")
+ output_options.add_argument('-q', '--quiet',
+ help='suppress warning messages',
+ action='store_true')
+ output_options.add_argument('-v', '--verbose',
+ help='print detailed information; use -vv for '
+ 'even more detail', action='count', default=0)
+ parser.add_argument('-f', '--force',
+ help='force overwriting existing files',
+ action='store_true')
+ parser.add_argument('-s', '--site', default=DEFAULT_SITE,
+ help='MediaWiki site to download from (default: '
+ '%(default)s)')
+ parser.add_argument('-o', '--output', help='write download to OUTPUT')
+ parser.add_argument('-a', '--batch',
+ help='treat FILE as a textfile containing multiple '
+ 'files to download, one URL or filename per line',
+ action='store_true')
args = parser.parse_args()
@@ -96,9 +96,10 @@ def main():
if args.verbose >= 1:
print("Info: using batch file '{}'".format(input_file))
try:
- fd = open(input_file, "r")
+ fd = open(input_file, 'r')
except IOError as e:
- print("File could not be read. The following error was encountered:")
+ print('File could not be read. The following error was '
+ 'encountered:')
print(e)
sys.exit(1)
else:
@@ -120,7 +121,8 @@ def download(dl, args):
site_name = url.netloc
if args.site is not DEFAULT_SITE and not args.quiet:
# this will work even if the user specifies 'en.wikipedia.org'
- print("Warning: target is a URL, ignoring site specified with --site")
+ print('Warning: target is a URL, ignoring site specified with '
+ '--site')
else:
filename = dl
site_name = args.site
@@ -130,7 +132,8 @@ def download(dl, args):
# check for valid site parameter
if not site_match:
- print("Only Wikimedia sites (wikipedia.org and wikimedia.org) are currently supported.")
+ print('Only Wikimedia sites (wikipedia.org and wikimedia.org) are '
+ 'currently supported.')
sys.exit(1)
# check if this is a valid file
@@ -139,13 +142,15 @@ def download(dl, args):
filename = file_match.group(2)
else:
# no file extension and/or prefix, probably an article
- print("Downloading Wikipedia articles is not currently supported.", end="")
+ print('Downloading Wikipedia articles is not currently supported.',
+ end='')
if file_match and not file_match.group(1):
# file extension detected, but no prefix
- # TODO: no longer possible to get to this point since file_match is None with no prefix
+ # TODO: no longer possible to get to this point since file_match is
+ # None with no prefix
print(" If this is a file, please add the 'File:' prefix.")
else:
- print("\n", end="")
+ print('\n', end='')
sys.exit(1)
filename = unquote(filename) # remove URL encoding for special characters
@@ -153,13 +158,14 @@ def download(dl, args):
dest = args.output or filename
if args.verbose >= 2:
- print("User agent: {}".format(USER_AGENT))
+ print('User agent: {}'.format(USER_AGENT))
# connect to site and identify ourselves
try:
site = Site(site_name, clients_useragent=USER_AGENT)
except ConnectionError:
- # usually this means there is no such site, or there's no network connection
+ # usually this means there is no such site, or there's no network
+ # connection
print("Error: couldn't connect to specified site.")
sys.exit(1)
except InvalidResponse as e:
@@ -172,31 +178,34 @@ def download(dl, args):
if file.imageinfo != {}:
# file exists either locally or at Wikimedia Commons
- file_url = file.imageinfo["url"]
- file_size = file.imageinfo["size"]
- file_sha1 = file.imageinfo["sha1"]
+ file_url = file.imageinfo['url']
+ file_size = file.imageinfo['size']
+ file_sha1 = file.imageinfo['sha1']
if args.verbose >= 1:
print("Info: downloading '{}' "
- "({} bytes) from {}".format(filename, file_size, site.host), end="")
+ '({} bytes) from {}'.format(filename, file_size, site.host),
+ end='')
if args.output:
print(" to '{}'".format(dest))
else:
- print("\n", end="")
- print("Info: {}".format(file_url))
+ print('\n', end='')
+ print('Info: {}'.format(file_url))
if os.path.isfile(dest) and not args.force:
- print("File '{}' already exists, skipping download (use -f to ignore)".format(dest))
+ print("File '{}' already exists, skipping download (use -f to "
+ "ignore)".format(dest))
else:
try:
- fd = open(dest, "wb")
+ fd = open(dest, 'wb')
except IOError as e:
- print("File could not be written. The following error was encountered:")
+ print('File could not be written. The following error was '
+ 'encountered:')
print(e)
sys.exit(1)
else:
# download the file
- with tqdm(total=file_size, unit="B",
+ with tqdm(total=file_size, unit='B',
unit_scale=True, unit_divisor=1024) as progress_bar:
with fd:
res = site.connection.get(file_url, stream=True)
@@ -209,57 +218,59 @@ def download(dl, args):
dl_sha1 = verify_hash(dest)
if args.verbose >= 1:
- print("Info: downloaded file SHA1 is {}".format(dl_sha1))
- print("Info: server file SHA1 is {}".format(file_sha1))
+ print('Info: downloaded file SHA1 is {}'.format(dl_sha1))
+ print('Info: server file SHA1 is {}'.format(file_sha1))
if dl_sha1 == file_sha1:
if args.verbose >= 1:
- print("Info: hashes match!")
+ print('Info: hashes match!')
# at this point, we've successfully downloaded the file
else:
- print("Error: hash mismatch! Downloaded file may be corrupt.")
+ print('Error: hash mismatch! Downloaded file may be corrupt.')
sys.exit(1)
else:
# no file information returned
- print("Target '{}' does not appear to be a valid file.".format(filename))
+ print("Target '{}' does not appear to be a valid file."
+ .format(filename))
sys.exit(1)
def valid_file(search_string):
"""
- Determines if the given string contains a valid file name, defined as a string
- ending with a '.' and at least one character, beginning with 'File:' or
- 'Image:', the standard file prefixes in MediaWiki.
+ Determines if the given string contains a valid file name, defined as a
+ string ending with a '.' and at least one character, beginning with 'File:'
+ or 'Image:', the standard file prefixes in MediaWiki.
:param search_string: string to validate
:returns: a regex Match object if there's a match or None otherwise
"""
# second group could also restrict to file extensions with three or more
# letters with ([^/\r\n\t\f\v]+\.\w{3,})
- file_regex = re.compile(r"(File:|Image:)([^/\r\n\t\f\v]+\.\w+)$", re.I)
+ file_regex = re.compile(r'(File:|Image:)([^/\r\n\t\f\v]+\.\w+)$', re.I)
return file_regex.search(search_string)
def valid_site(search_string):
"""
- Determines if the given string contains a valid site name, defined as a string
- ending with 'wikipedia.org' or 'wikimedia.org'. This covers all subdomains of
- those domains. Eventually, it should be possible to support any MediaWiki site,
- regardless of domain name.
+ Determines if the given string contains a valid site name, defined as a
+ string ending with 'wikipedia.org' or 'wikimedia.org'. This covers all
+ subdomains of those domains. Eventually, it should be possible to support
+ any MediaWiki site, regardless of domain name.
:param search_string: string to validate
:returns: a regex Match object if there's a match or None otherwise
"""
- site_regex = re.compile(r"wiki[mp]edia\.org$", re.I)
+ site_regex = re.compile(r'wiki[mp]edia\.org$', re.I)
return site_regex.search(search_string)
def verify_hash(filename):
"""
- Calculates the SHA1 hash of the given file for comparison with a known value.
+ Calculates the SHA1 hash of the given file for comparison with a known
+ value.
:param filename: name of the file to calculate a hash for
:return: hash digest
"""
hasher = hashlib.sha1()
- with open(filename, "rb") as dl:
+ with open(filename, 'rb') as dl:
buf = dl.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)