aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/wikiget/__init__.py9
-rw-r--r--src/wikiget/dl.py26
-rw-r--r--src/wikiget/validations.py4
-rw-r--r--src/wikiget/wikiget.py137
4 files changed, 105 insertions, 71 deletions
diff --git a/src/wikiget/__init__.py b/src/wikiget/__init__.py
index 4adcae3..20ea620 100644
--- a/src/wikiget/__init__.py
+++ b/src/wikiget/__init__.py
@@ -17,12 +17,15 @@
from mwclient import __version__ as mwclient_version
-from .version import __version__ as wikiget_version
+from wikiget.version import __version__ as wikiget_version
# set some global constants
BLOCKSIZE = 65536
CHUNKSIZE = 1024
DEFAULT_SITE = "commons.wikimedia.org"
DEFAULT_PATH = "/w/"
-USER_AGENT = (f"wikiget/{wikiget_version} (https://github.com/clpo13/wikiget) "
- f"mwclient/{mwclient_version}")
+USER_AGENT = "wikiget/{} (https://github.com/clpo13/wikiget) mwclient/{}".format(
+ wikiget_version, mwclient_version
+)
+STD_VERBOSE = 1
+VERY_VERBOSE = 2
diff --git a/src/wikiget/dl.py b/src/wikiget/dl.py
index 8f32218..9850ce8 100644
--- a/src/wikiget/dl.py
+++ b/src/wikiget/dl.py
@@ -24,8 +24,8 @@ from mwclient import APIError, InvalidResponse, LoginError, Site
from requests import ConnectionError, HTTPError
from tqdm import tqdm
-from . import CHUNKSIZE, DEFAULT_SITE, USER_AGENT
-from .validations import valid_file, verify_hash
+import wikiget
+from wikiget.validations import valid_file, verify_hash
def download(dl, args):
@@ -34,7 +34,7 @@ def download(dl, args):
if url.netloc:
filename = url.path
site_name = url.netloc
- if args.site is not DEFAULT_SITE:
+ if args.site is not wikiget.DEFAULT_SITE:
# this will work even if the user specifies 'commons.wikimedia.org'
logging.warning("target is a URL, "
"ignoring site specified with --site")
@@ -57,12 +57,12 @@ def download(dl, args):
dest = args.output or filename
- logging.debug(f"User agent: {USER_AGENT}")
+ logging.debug(f"User agent: {wikiget.USER_AGENT}")
# connect to site and identify ourselves
logging.info(f"Site name: {site_name}")
try:
- site = Site(site_name, path=args.path, clients_useragent=USER_AGENT)
+ site = Site(site_name, path=args.path, clients_useragent=wikiget.USER_AGENT)
if args.username and args.password:
site.login(args.username, args.password)
except ConnectionError as e:
@@ -119,24 +119,28 @@ def download(dl, args):
else:
try:
fd = open(dest, "wb")
- except IOError as e:
+ except OSError as e:
logging.error("File could not be written. "
"The following error was encountered:")
logging.error(e)
sys.exit(1)
else:
# download the file(s)
- if args.verbose >= 1:
+ if args.verbose >= wikiget.STD_VERBOSE:
leave_bars = True
else:
leave_bars = False
- with tqdm(leave=leave_bars, total=file_size,
- unit="B", unit_scale=True,
- unit_divisor=CHUNKSIZE) as progress_bar:
+ with tqdm(
+ leave=leave_bars,
+ total=file_size,
+ unit="B",
+ unit_scale=True,
+ unit_divisor=wikiget.CHUNKSIZE,
+ ) as progress_bar:
with fd:
res = site.connection.get(file_url, stream=True)
progress_bar.set_postfix(file=dest, refresh=False)
- for chunk in res.iter_content(CHUNKSIZE):
+ for chunk in res.iter_content(wikiget.CHUNKSIZE):
fd.write(chunk)
progress_bar.update(len(chunk))
diff --git a/src/wikiget/validations.py b/src/wikiget/validations.py
index bd99570..dc70df4 100644
--- a/src/wikiget/validations.py
+++ b/src/wikiget/validations.py
@@ -18,7 +18,7 @@
import hashlib
import re
-from . import BLOCKSIZE
+from wikiget import BLOCKSIZE
def valid_file(search_string):
@@ -55,7 +55,7 @@ def verify_hash(filename):
:param filename: name of the file to calculate a hash for
:return: hash digest
"""
- hasher = hashlib.sha1()
+ hasher = hashlib.sha1() # noqa: S324
with open(filename, "rb") as dl:
buf = dl.read(BLOCKSIZE)
while len(buf) > 0:
diff --git a/src/wikiget/wikiget.py b/src/wikiget/wikiget.py
index a8679c9..b9a227f 100644
--- a/src/wikiget/wikiget.py
+++ b/src/wikiget/wikiget.py
@@ -19,8 +19,8 @@ import argparse
import logging
import sys
-from . import DEFAULT_SITE, DEFAULT_PATH, wikiget_version
-from .dl import download
+import wikiget
+from wikiget.dl import download
def main():
@@ -29,65 +29,92 @@ def main():
when installed with `pip install` or `python setup.py install`.
"""
- parser = argparse.ArgumentParser(description="""
- A tool for downloading files from
- MediaWiki sites using the file name or
- description page URL
- """,
- epilog="""
- Copyright (C) 2018-2021 Cody Logan
- and contributors.
- License GPLv3+: GNU GPL version 3 or later
- <http://www.gnu.org/licenses/gpl.html>.
- This is free software; you are free to
- change and redistribute it under certain
- conditions. There is NO WARRANTY, to the
- extent permitted by law.
- """)
- parser.add_argument("FILE", help="""
- name of the file to download with the File:
- prefix, or the URL of its file description page
- """)
- parser.add_argument("-V", "--version", action="version",
- version=f"%(prog)s {wikiget_version}")
+ parser = argparse.ArgumentParser(
+ description="""
+ A tool for downloading files from
+ MediaWiki sites using the file name or
+ description page URL
+ """,
+ epilog="""
+ Copyright (C) 2018-2023 Cody Logan
+ and contributors.
+ License GPLv3+: GNU GPL version 3 or later
+ <http://www.gnu.org/licenses/gpl.html>.
+ This is free software; you are free to
+ change and redistribute it under certain
+ conditions. There is NO WARRANTY, to the
+ extent permitted by law.
+ """,
+ )
+ parser.add_argument(
+ "FILE",
+ help="""
+ name of the file to download with the File:
+ prefix, or the URL of its file description page
+ """,
+ )
+ parser.add_argument(
+ "-V",
+ "--version",
+ action="version",
+ version=f"%(prog)s {wikiget.wikiget_version}",
+ )
message_options = parser.add_mutually_exclusive_group()
- message_options.add_argument("-q", "--quiet",
- help="suppress warning messages",
- action="store_true")
- message_options.add_argument("-v", "--verbose",
- help="print detailed information; "
- "use -vv for even more detail",
- action="count", default=0)
- parser.add_argument("-f", "--force",
- help="force overwriting existing files",
- action="store_true")
- parser.add_argument("-s", "--site", default=DEFAULT_SITE,
- help="MediaWiki site to download from "
- "(default: %(default)s)")
- parser.add_argument("-p", "--path", default=DEFAULT_PATH,
- help="MediaWiki site path, where api.php is located "
- "(default: %(default)s)")
- parser.add_argument("--username", default="",
- help="MediaWiki site username, for private wikis")
- parser.add_argument("--password", default="",
- help="MediaWiki site password, for private wikis")
+ message_options.add_argument(
+ "-q", "--quiet", help="suppress warning messages", action="store_true"
+ )
+ message_options.add_argument(
+ "-v",
+ "--verbose",
+ help="print detailed information; use -vv for even more detail",
+ action="count",
+ default=0,
+ )
+ parser.add_argument(
+ "-f", "--force", help="force overwriting existing files", action="store_true"
+ )
+ parser.add_argument(
+ "-s",
+ "--site",
+ default=wikiget.DEFAULT_SITE,
+ help="MediaWiki site to download from (default: %(default)s)",
+ )
+ parser.add_argument(
+ "-p",
+ "--path",
+ default=wikiget.DEFAULT_PATH,
+ help="MediaWiki site path, where api.php is located (default: %(default)s)",
+ )
+ parser.add_argument(
+ "--username", default="", help="MediaWiki site username, for private wikis"
+ )
+ parser.add_argument(
+ "--password", default="", help="MediaWiki site password, for private wikis"
+ )
output_options = parser.add_mutually_exclusive_group()
- output_options.add_argument("-o", "--output",
- help="write download to OUTPUT")
- output_options.add_argument("-a", "--batch",
- help="treat FILE as a textfile containing "
- "multiple files to download, one URL or "
- "filename per line", action="store_true")
- parser.add_argument("-l", "--logfile", default="",
- help="save log output to LOGFILE")
+ output_options.add_argument("-o", "--output", help="write download to OUTPUT")
+ output_options.add_argument(
+ "-a",
+ "--batch",
+ help="treat FILE as a textfile containing "
+ "multiple files to download, one URL or "
+ "filename per line",
+ action="store_true",
+ )
+ parser.add_argument(
+ "-l",
+ "--logfile",
+ default="",
+ help="save log output to LOGFILE"
+ )
args = parser.parse_args()
loglevel = logging.WARNING
- if args.verbose >= 2:
+ if args.verbose >= wikiget.VERY_VERBOSE:
# this includes API and library messages
loglevel = logging.DEBUG
- elif args.verbose >= 1:
+ elif args.verbose >= wikiget.STD_VERBOSE:
loglevel = logging.INFO
elif args.quiet:
loglevel = logging.ERROR
@@ -120,7 +147,7 @@ def main():
# log events are appended to the file if it already exists,
# so note the start of a new download session
- logging.info(f"Starting download session using wikiget {wikiget_version}")
+ logging.info(f"Starting download session using wikiget {wikiget.wikiget_version}")
# logging.info(f"Log level is set to {loglevel}")
if args.batch:
@@ -132,7 +159,7 @@ def main():
try:
fd = open(input_file, "r")
- except IOError as e:
+ except OSError as e:
logging.error("File could not be read. "
"The following error was encountered:")
logging.error(e)
generated by cgit v1.2.3 (git 2.25.1) at 2026-02-12 02:59:23 +0000