1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
|
# wikiget - CLI tool for downloading files from Wikimedia sites
# Copyright (C) 2018-2023 Cody Logan and contributors
# SPDX-License-Identifier: GPL-3.0-or-later
#
# Wikiget is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Wikiget is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Wikiget. If not, see <https://www.gnu.org/licenses/>.
import argparse
import logging
import sys
from concurrent.futures import ThreadPoolExecutor
import wikiget
from wikiget.dl import download, prep_download
def construct_parser():
parser = argparse.ArgumentParser(
description="""
A tool for downloading files from MediaWiki sites using the file name or
description page URL
""",
epilog="""
Copyright (C) 2018-2023 Cody Logan and contributors. License GPLv3+: GNU GPL
version 3 or later <http://www.gnu.org/licenses/gpl.html>. This is free
software; you are free to change and redistribute it under certain conditions.
There is NO WARRANTY, to the extent permitted by law.
""",
)
parser.add_argument(
"FILE",
help="""
name of the file to download with the File: prefix, or the URL of its file
description page
""",
)
parser.add_argument(
"-V",
"--version",
action="version",
version=f"%(prog)s {wikiget.wikiget_version}",
)
message_options = parser.add_mutually_exclusive_group()
message_options.add_argument(
"-q", "--quiet", help="suppress warning messages", action="store_true"
)
message_options.add_argument(
"-v",
"--verbose",
help="print detailed information; use -vv for even more detail",
action="count",
default=0,
)
parser.add_argument(
"-f", "--force", help="force overwriting existing files", action="store_true"
)
parser.add_argument(
"-s",
"--site",
default=wikiget.DEFAULT_SITE,
help="MediaWiki site to download from (default: %(default)s)",
)
parser.add_argument(
"-P",
"--path",
default=wikiget.DEFAULT_PATH,
help="MediaWiki site path, where api.php is located (default: %(default)s)",
)
parser.add_argument(
"-u",
"--username",
default="",
help="MediaWiki site username, for private wikis",
)
parser.add_argument(
"-p",
"--password",
default="",
help="MediaWiki site password, for private wikis",
)
output_options = parser.add_mutually_exclusive_group()
output_options.add_argument("-o", "--output", help="write download to OUTPUT")
output_options.add_argument(
"-a",
"--batch",
help="treat FILE as a textfile containing multiple files to download, one URL "
"or filename per line",
action="store_true",
)
parser.add_argument(
"-l", "--logfile", default="", help="save log output to LOGFILE"
)
parser.add_argument(
"-j",
"--threads",
default=1,
help="number of parallel downloads to attempt in batch mode",
type=int,
)
return parser
def configure_logging(args):
loglevel = logging.WARNING
if args.verbose >= wikiget.VERY_VERBOSE:
# this includes API and library messages
loglevel = logging.DEBUG
elif args.verbose >= wikiget.STD_VERBOSE:
loglevel = logging.INFO
elif args.quiet:
loglevel = logging.ERROR
# configure logging:
# console log level is set via -v, -vv, and -q options;
# file log level is always debug (TODO: make this user configurable)
base_format = "%(threadName)s - %(message)s"
log_format = "[%(levelname)s] " + base_format
if args.logfile:
# log to console and file
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s [%(levelname)-7s] " + base_format,
filename=args.logfile,
)
console = logging.StreamHandler()
console.setLevel(loglevel)
console.setFormatter(logging.Formatter(log_format))
logging.getLogger("").addHandler(console)
else:
# log only to console
logging.basicConfig(level=loglevel, format=log_format)
def batch_download(args):
input_file = args.FILE
dl_list = []
logging.info(f"Using batch file '{input_file}'.")
try:
fd = open(input_file)
except OSError as e:
logging.error("File could not be read. The following error was encountered:")
logging.error(e)
sys.exit(1)
else:
with fd:
# store file contents in memory in case something happens to the file
# while we're downloading
for _, line in enumerate(fd):
dl_list.append(line)
# TODO: validate file contents before download process starts
with ThreadPoolExecutor(
max_workers=args.threads,
thread_name_prefix="download",
) as executor:
futures = []
for line_num, line in enumerate(dl_list, start=1):
url = line.strip()
# keep track of batch file line numbers for debugging/logging purposes
logging.info(f"Downloading '{url}' at line {line_num}:")
file = prep_download(url, args)
future = executor.submit(download, file, args)
futures.append(future)
# wait for downloads to finish
for future in futures:
future.result()
def main():
# setup
parser = construct_parser()
args = parser.parse_args()
configure_logging(args)
# log events are appended to the file if it already exists, so note the start of a
# new download session
logging.info(f"Starting download session using wikiget {wikiget.wikiget_version}")
if args.batch:
# batch download mode
batch_download(args)
else:
# single download mode
file = prep_download(args.FILE, args)
download(file, args)
|
