aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorclpo13 <cody@lokken.dev>2023-10-20 16:57:32 -0700
committerGitHub <noreply@github.com>2023-10-20 16:57:32 -0700
commit8583862e2d16144f184db2e31dbc37dbe3464fed (patch)
tree4a0d9edb5301b26d9dbd22ceb307a7e3b1db4820
parente274ccea56219c7d07c0e677d44c8122a699dcaf (diff)
parentc1820026f97eaf671c29ab30f02879de0ac4df89 (diff)
downloadwikiget-8583862e2d16144f184db2e31dbc37dbe3464fed.tar.gz
wikiget-8583862e2d16144f184db2e31dbc37dbe3464fed.zip
Merge pull request #8 from clpo13/dev
Merge dev branch changes into master
-rw-r--r--.github/workflows/python.yml20
-rw-r--r--.gitignore6
-rw-r--r--MANIFEST.in5
-rw-r--r--README.md112
-rw-r--r--docs/Makefile6
-rw-r--r--docs/wikiget.1123
-rw-r--r--docs/wikiget.1.md116
-rw-r--r--pyproject.toml68
-rw-r--r--setup.py17
-rw-r--r--src/wikiget/__init__.py7
-rw-r--r--src/wikiget/dl.py219
-rw-r--r--src/wikiget/exceptions.py20
-rw-r--r--src/wikiget/file.py39
-rw-r--r--src/wikiget/logging.py58
-rw-r--r--src/wikiget/parse.py59
-rw-r--r--src/wikiget/validations.py35
-rw-r--r--src/wikiget/version.py17
-rw-r--r--src/wikiget/wikiget.py109
-rw-r--r--tests/test_dl.py46
-rw-r--r--tests/test_file_class.py37
-rw-r--r--tests/test_parse.py60
-rw-r--r--tests/test_validations.py2
22 files changed, 928 insertions, 253 deletions
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index bef2670..7f44810 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -5,9 +5,9 @@ name: Python package
on:
push:
- branches: [ "master" ]
+ branches: [ "dev" ]
pull_request:
- branches: [ "master" ]
+ branches: [ "master", "dev" ]
jobs:
build:
@@ -16,7 +16,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+ python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v3
@@ -27,12 +27,20 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip setuptools wheel
- python -m pip install flake8 pytest pytest-cov
+ python -m pip install flake8 pytest "coverage[toml]"
python -m pip install .
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
- flake8 . --count --show-source --statistics
+ flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
- name: Test with pytest
run: |
- pytest
+ coverage run -m pytest
+ - name: Convert coverage file to XML
+ run: |
+ coverage combine
+ coverage xml
+ - name: Upload coverage reports to Codecov
+ uses: codecov/codecov-action@v3
+ env:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.gitignore b/.gitignore
index 3d6c39b..308c5e5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -107,3 +107,9 @@ venv.bak/
.vs/
.vscode/
.idea/
+
+# downloaded images, batch test files
+*.jpg
+*.jpeg
+*.png
+batch.txt \ No newline at end of file
diff --git a/MANIFEST.in b/MANIFEST.in
index 7592f0e..429c6c8 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,3 +1,4 @@
-graft wikiget
-graft test
+graft src
+graft tests
+graft docs
global-exclude *.py[cod]
diff --git a/README.md b/README.md
index a0ccf8d..bd8ece2 100644
--- a/README.md
+++ b/README.md
@@ -3,47 +3,45 @@
[![Python package](https://github.com/clpo13/wikiget/actions/workflows/python.yml/badge.svg?branch=master)](https://github.com/clpo13/wikiget/actions/workflows/python.yml)
[![PyPI version](https://badge.fury.io/py/wikiget.svg)](https://badge.fury.io/py/wikiget)
-Something like wget for downloading a file from MediaWiki sites (like Wikipedia
-or Wikimedia Commons) using only the file name or the URL of its description
-page.
+Something like wget for downloading a file from MediaWiki sites (like Wikipedia or Wikimedia Commons) using only the
+file name or the URL of its description page.
-Requires Python 3.7+. Get it with `pip install --user wikiget` or `pipx install wikiget`.
+Requires Python 3.7+ and pip. Get it with `pip install wikiget` or `pipx install wikiget`. For the latest features, at
+the risk of bugs and undocumented behavior, you can try the dev branch:
+`pip install https://github.com/clpo13/wikiget/archive/refs/heads/dev.zip`
## Usage
-`wikiget [-h] [-V] [-q | -v] [-f] [-s SITE] [-p PATH] [--username USERNAME]
-[--password PASSWORD] [-o OUTPUT | -a] FILE`
-
-If `FILE` is in the form `File:Example.jpg` or `Image:Example.jpg`, it will be
-fetched from the default site, which is "commons.wikimedia.org". If it's the
-fully-qualified URL of a file description page, like
-`https://en.wikipedia.org/wiki/File:Example.jpg`, the file is fetched from the
-specified site, in this case "en.wikipedia.org". Full URLs may contain
-characters your shell interprets differently, so you can either escape those
-characters with a backslash `\` or surround the entire URL with single `'` or
-double `"` quotes. Use of a fully-qualified URL like this may require setting
-the `--path` flag (see next paragraph).
-
-The site can also be specified with the `--site` flag, though this will not have
-any effect if the full URL is given. Non-Wikimedia sites should work, but you
-may need to specify the wiki's script path with `--path` (where `index.php` and
-`api.php` live; on Wikimedia sites it's `/w/`, but other sites may use `/` or
-something else entirely). Private wikis (those requiring login even for read
-access) are also supported with the use of the `--username` and `--password`
-flags.
-
-More detailed information, such as the site used and full URL of the file, can
-be displayed with `-v` or `--verbose`. Use `-vv` to display even more detail.
-`-q` can be used to silence warnings.
-
-By default, the program won't overwrite existing files with the same name as the
-target, but this can be forced with `-f` or `--force`. Additionally, the file
-can be downloaded to a different name with `-o`.
-
-Files can be batch downloaded with the `-a` or `--batch` flag. In this mode,
-`FILE` will be treated as an input file containing multiple files to download,
-one filename or URL per line. If an error is encountered, execution stops
-immediately and the offending filename is printed.
+`wikiget [-h] [-V] [-q | -v] [-f] [-s SITE] [-P PATH] [-u USERNAME] [-p PASSWORD] [-o OUTPUT | -a] [-l LOGFILE] [-j THREADS] FILE`
+
+The only required parameter is `FILE`, which is the file you want to download. It can either be the name of the file on
+the wiki, including the namespace prefix, or a link to the file description page. If `FILE` is in the form
+`File:Example.jpg` or `Image:Example.jpg`, it will be fetched from the default site, which is "commons.wikimedia.org".
+If it's the fully-qualified URL of a file description page, like `https://en.wikipedia.org/wiki/File:Example.jpg`, the
+file is fetched from the site in the URL, in this case "en.wikipedia.org". Note: full URLs may contain characters your
+shell interprets differently, so you can either escape those characters with a backslash `\` or surround the entire URL
+with single `'` or double `"` quotes. Use of a fully-qualified URL like this may require setting the `--path` flag (see
+next paragraph).
+
+The site can also be specified with the `--site` flag, though this will not have any effect if the full URL is given.
+Non-Wikimedia sites should work, but you may need to specify the wiki's script path with `--path` (where `index.php` and
+`api.php` live; on Wikimedia sites it's `/w/`, but other sites may use `/` or something else entirely). Private wikis
+(those requiring login even for read access) are also supported with the use of the `--username` and `--password` flags.
+
+More detailed information, such as the site used and full URL of the file, can be displayed with `-v` or `--verbose`.
+Use `-vv` to display even more detail, mainly debugging information or API messages. `-q` can be used to silence
+warnings. A logfile can be specified with `-l` or `--logfile`. If this option is present, the logfile will contain the
+same information as `-v` along with timestamps. New log entries will be appended to an existing logfile.
+
+By default, the program won't overwrite existing files with the same name as the target, but this can be forced with
+`-f` or `--force`. Additionally, the file can be downloaded to a different name with `-o`.
+
+Files can be batch downloaded with the `-a` or `--batch` flag. In this mode, `FILE` will be treated as an input file
+containing multiple files to download, one filename or URL per line. Blank lines and lines starting with "#" are
+ignored. If an error is encountered, execution stops immediately and the offending filename is printed. For large
+batches, the process can be sped up by downloading files in parallel. The number of parallel downloads can be set with
+`-j`. For instance, with `-a -j4`, wikiget will download four files at once. Without `-j` or with `-j` by itself without
+a number, wikiget will download the files one at a time.
### Example usage
@@ -62,17 +60,15 @@ wikiget https://en.wikipedia.org/wiki/File:Example.jpg -o test.jpg
## Contributing
-Pull requests or bug reports are more than welcome.
+Pull requests, bug reports, or feature requests are more than welcome.
It's recommended that you use a
[virtual environment manager](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/)
-like venv or [virtualenv](https://virtualenv.pypa.io/en/latest/) to create an
-isolated environment in which to install this package's dependencies so as not
-to clutter your system Python environment:
+like venv or [virtualenv](https://virtualenv.pypa.io/en/latest/) to create an isolated environment in which to install
+this package's dependencies so as not to clutter your system Python environment:
```bash
-# if you plan on submitting pull requests, fork the repo on GitHub
-# and clone that instead
+# if you plan on submitting pull requests, fork the repo on GitHub and clone that instead
git clone https://github.com/clpo13/wikiget
cd wikiget
@@ -92,29 +88,23 @@ source venv/bin/activate
```
Then run `pip install -e .` to invoke an
-["editable" install](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs),
-meaning any changes made to the source will be reflected immediately in the executable
-script. Unit tests can be run with `pytest` (make sure to run `pip install pytest`
-in the virtual environment first.)
+["editable" install](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs)
+meaning any changes made to the source will be reflected immediately in the executable script. Unit tests can be run
+with `pytest` (make sure to run `pip install pytest` in the virtual environment first.)
-Alternatively, using [Hatch](https://hatch.pypa.io/latest/), simply clone the repository
-and run `hatch run test` to create the environment and run pytest. Also try `hatch shell`
-or `hatch run wikiget --help`.
+Alternatively, using [Hatch](https://hatch.pypa.io/latest/), simply clone the repository and run `hatch run test` to
+create the environment and run pytest. Also try `hatch shell` or `hatch run wikiget --help`.
## License
Copyright (C) 2018-2023 Cody Logan and contributors
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public
+License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
+version.
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-You should have received a copy of the GNU General Public License
-along with this program (see [LICENSE](LICENSE)). If not, see
-<https://www.gnu.org/licenses/>.
+You should have received a copy of the GNU General Public License along with this program (see [LICENSE](LICENSE)).
+If not, see <https://www.gnu.org/licenses/>.
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..6ce62df
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,6 @@
+.PHONY: man
+
+man: wikiget.1
+
+wikiget.1: wikiget.1.md
+ pandoc -s -f markdown -t man -o wikiget.1 wikiget.1.md
diff --git a/docs/wikiget.1 b/docs/wikiget.1
new file mode 100644
index 0000000..fa1a33d
--- /dev/null
+++ b/docs/wikiget.1
@@ -0,0 +1,123 @@
+.\" Automatically generated by Pandoc 3.1.8
+.\"
+.TH "WIKIGET" "1" "October 5, 2023" "Version 0.5.1" "Wikiget User Manual"
+.SH NAME
+wikiget - download files from MediaWiki sites
+.SH SYNOPSIS
+.PP
+\f[B]wikiget\f[R] [\f[I]options\f[R]] \f[I]FILE\f[R]
+.PD 0
+.P
+.PD
+\f[B]wikiget\f[R] [\f[I]options\f[R]] [-\f[B]a\f[R]|--\f[B]batch\f[R]]
+\f[I]BATCHFILE\f[R]
+.PD 0
+.P
+.PD
+\f[B]wikiget\f[R] [-\f[B]V\f[R]|--\f[B]version\f[R]]
+.PD 0
+.P
+.PD
+\f[B]wikiget\f[R] [-\f[B]h\f[R]|--\f[B]help\f[R]]
+.SH DESCRIPTION
+Something like \f[B]wget\f[R](1) for downloading a file from MediaWiki
+sites (like Wikipedia or Wikimedia Commons) using only the file name or
+the URL of its description page.
+.SH OPTIONS
+.TP
+\f[I]FILE\f[R]
+The file to be downloaded.
+If \f[I]FILE\f[R] is in the form \f[I]File:Example.jpg\f[R] or
+\f[I]Image:Example.jpg\f[R], it will be fetched from the default site,
+which is \[lq]commons.wikimedia.org\[rq].
+If it\[cq]s the fully-qualified URL of a file description page, like
+\f[I]https://en.wikipedia.org/wiki/File:Example.jpg\f[R], the file is
+fetched from the site in the URL, in this case
+\[lq]en.wikipedia.org\[rq].
+.TP
+\f[I]BATCHFILE\f[R]
+In batch download mode (activated with -\f[B]a\f[R] or
+--\f[B]batch\f[R]), this is a text file containing multiple file names
+or URLs to be downloaded, one per line.
+Blank lines and lines starting with \[lq]#\[rq] are ignored.
+If an error is encountered during download, execution stops immediately
+and the offending filename is printed.
+.TP
+-\f[B]s\f[R], --\f[B]site\f[R] \f[I]SITE\f[R]
+MediaWiki site to download from.
+Will not have any effect if the full URL is given in the \f[I]FILE\f[R]
+parameter.
+.TP
+-\f[B]P\f[R], --\f[B]path\f[R] \f[I]PATH\f[R]
+Script path for the wiki, where \[lq]index.php\[rq] and
+\[lq]api.php\[rq] live.
+On Wikimedia sites, it\[cq]s \[lq]/w/\[rq], the default, but other sites
+may use \[lq]/\[rq] or something else entirely.
+.TP
+-\f[B]u\f[R], --\f[B]username\f[R] \f[I]USERNAME\f[R]
+Username for private wikis that require a login even for read access.
+.TP
+-\f[B]p\f[R], --\f[B]password\f[R] \f[I]PASSWORD\f[R]
+Password for private wikis that require a login even for read access.
+.TP
+-\f[B]o\f[R], --\f[B]output\f[R] \f[I]OUTPUT\f[R]
+By default, the output filename is the same as the remote filename
+(without the File: or Image: prefix), but this can be changed with this
+option.
+.TP
+-\f[B]l\f[R], --\f[B]logfile\f[R] \f[I]LOGFILE\f[R]
+Specify a logfile, which will contain detailed information about the
+download process.
+If the logfile already exists, new log information is appended to it.
+.TP
+-\f[B]f\f[R], --\f[B]force\f[R]
+Force existing files to be overwritten.
+.TP
+-\f[B]a\f[R], --\f[B]batch\f[R]
+If this flag is set, \f[B]wikiget\f[R] will run in batch download mode
+(see \f[I]BATCHFILE\f[R]).
+.TP
+-\f[B]j\f[R], --\f[B]threads\f[R]
+Number of parallel downloads to attempt in batch mode.
+This option has no effect if -\f[B]a\f[R] is not also set.
+.TP
+-\f[B]v\f[R], --\f[B]verbose\f[R]
+Print additional information, such as the site used and the full URL of
+the file.
+Additional invocations will increase the level of detail.
+.TP
+-\f[B]q\f[R], --\f[B]quiet\f[R]
+Silence warnings and minimize printed output.
+.TP
+-\f[B]V\f[R], --\f[B]version\f[R]
+Print the version number of the program.
+.TP
+-\f[B]h\f[R], --\f[B]help\f[R]
+Print a brief summary of these options.
+.SH EXAMPLES
+.IP
+.EX
+wikiget File:Example.jpg
+wikiget --site en.wikipedia.org File:Example.jpg
+wikiget https://en.wikipedia.org/wiki/File:Example.jpg -o test.jpg
+.EE
+.SH BUG REPORTS
+https://github.com/clpo13/wikiget/issues
+.SH LICENSE
+Copyright (C) 2018-2023 Cody Logan and contributors
+.PP
+This program is free software: you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the
+Free Software Foundation, either version 3 of the License, or (at your
+option) any later version.
+.PP
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+See the GNU General Public License for more details.
+.PP
+You should have received a copy of the GNU General Public License along
+with this program.
+If not, see https://www.gnu.org/licenses/.
+.SH AUTHORS
+Cody Logan <clpo13@gmail.com>.
diff --git a/docs/wikiget.1.md b/docs/wikiget.1.md
new file mode 100644
index 0000000..fd274d5
--- /dev/null
+++ b/docs/wikiget.1.md
@@ -0,0 +1,116 @@
+% WIKIGET(1) Version 0.5.1 | Wikiget User Manual
+% Cody Logan <clpo13@gmail.com>
+% October 5, 2023
+
+# NAME
+
+wikiget - download files from MediaWiki sites
+
+# SYNOPSIS
+
+| **wikiget** \[*options*] *FILE*
+| **wikiget** \[*options*] \[\-**a**|\-\-**batch**] *BATCHFILE*
+| **wikiget** \[\-**V**|\-\-**version**]
+| **wikiget** \[\-**h**|\-\-**help**]
+
+# DESCRIPTION
+
+Something like **wget**(1) for downloading a file from MediaWiki sites (like Wikipedia or Wikimedia Commons) using only
+the file name or the URL of its description page.
+
+# OPTIONS
+
+*FILE*
+
+: The file to be downloaded. If *FILE* is in the form *File:Example.jpg* or *Image:Example.jpg*, it will be fetched
+ from the default site, which is "commons.wikimedia.org". If it's the fully-qualified URL of a file description page,
+ like *https://en.wikipedia.org/wiki/File:Example.jpg*, the file is fetched from the site in the URL, in this case
+ "en.wikipedia.org".
+
+*BATCHFILE*
+
+: In batch download mode (activated with \-**a** or \-\-**batch**), this is a text file containing multiple file names
+ or URLs to be downloaded, one per line. Blank lines and lines starting with "#" are ignored. If an error is
+ encountered during download, execution stops immediately and the offending filename is printed.
+
+\-**s**, \-\-**site** *SITE*
+
+: MediaWiki site to download from. Will not have any effect if the full URL is given in the *FILE* parameter.
+
+\-**P**, \-\-**path** *PATH*
+
+: Script path for the wiki, where "index.php" and "api.php" live. On Wikimedia sites, it's "/w/", the default, but
+ other sites may use "/" or something else entirely.
+
+\-**u**, \-\-**username** *USERNAME*
+
+: Username for private wikis that require a login even for read access.
+
+\-**p**, \-\-**password** *PASSWORD*
+
+: Password for private wikis that require a login even for read access.
+
+\-**o**, \-\-**output** *OUTPUT*
+
+: By default, the output filename is the same as the remote filename (without the File: or Image: prefix), but this
+ can be changed with this option.
+
+\-**l**, \-\-**logfile** *LOGFILE*
+
+: Specify a logfile, which will contain detailed information about the download process. If the logfile already
+ exists, new log information is appended to it.
+
+\-**f**, \-\-**force**
+
+: Force existing files to be overwritten.
+
+\-**a**, \-\-**batch**
+
+: If this flag is set, **wikiget** will run in batch download mode (see *BATCHFILE*).
+
+\-**j**, \-\-**threads**
+
+: Number of parallel downloads to attempt in batch mode. This option has no effect if \-**a** is not also set.
+
+\-**v**, \-\-**verbose**
+
+: Print additional information, such as the site used and the full URL of the file. Additional invocations will
+ increase the level of detail.
+
+\-**q**, \-\-**quiet**
+
+: Silence warnings and minimize printed output.
+
+\-**V**, \-\-**version**
+
+: Print the version number of the program.
+
+\-**h**, \-\-**help**
+
+: Print a brief summary of these options.
+
+# EXAMPLES
+
+```
+wikiget File:Example.jpg
+wikiget --site en.wikipedia.org File:Example.jpg
+wikiget https://en.wikipedia.org/wiki/File:Example.jpg -o test.jpg
+```
+
+# BUG REPORTS
+
+https://github.com/clpo13/wikiget/issues
+
+# LICENSE
+
+Copyright (C) 2018-2023 Cody Logan and contributors
+
+This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public
+License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with this program. If not, see
+https://www.gnu.org/licenses/.
diff --git a/pyproject.toml b/pyproject.toml
index aab4b3f..11fcaad 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,36 +8,37 @@ dynamic = ["version"]
description = "CLI tool for downloading files from MediaWiki sites"
readme = "README.md"
authors = [
- {name = "Cody Logan", email = "clpo13@gmail.com"}
+ {name = "Cody Logan", email = "clpo13@gmail.com"}
]
requires-python = ">=3.7"
license = {text = "GPL-3.0-or-later"}
keywords = ["commons", "mediawiki", "wikimedia", "wikipedia"]
classifiers = [
- "Development Status :: 4 - Beta",
- "Environment :: Console",
- "Intended Audience :: End Users/Desktop",
- "Operating System :: OS Independent",
- "Topic :: Internet",
- "Topic :: Internet :: WWW/HTTP",
- "Topic :: Multimedia",
- "Topic :: Multimedia :: Graphics",
- "Topic :: Multimedia :: Sound/Audio",
- "Topic :: Multimedia :: Video",
- "Topic :: Utilities",
- "Programming Language :: Python",
- "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3 :: Only",
- "Programming Language :: Python :: 3.7",
- "Programming Language :: Python :: 3.8",
- "Programming Language :: Python :: 3.9",
- "Programming Language :: Python :: 3.10",
- "Programming Language :: Python :: 3.11",
+ "Development Status :: 4 - Beta",
+ "Environment :: Console",
+ "Intended Audience :: End Users/Desktop",
+ "Operating System :: OS Independent",
+ "Topic :: Internet",
+ "Topic :: Internet :: WWW/HTTP",
+ "Topic :: Multimedia",
+ "Topic :: Multimedia :: Graphics",
+ "Topic :: Multimedia :: Sound/Audio",
+ "Topic :: Multimedia :: Video",
+ "Topic :: Utilities",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3 :: Only",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
]
dependencies = [
- "mwclient>=0.10.0",
- "requests",
- "tqdm",
+ "mwclient>=0.10.0",
+ "requests",
+ "tqdm",
]
[project.urls]
@@ -52,15 +53,20 @@ path = "src/wikiget/version.py"
[tool.pytest.ini_options]
addopts = [
- "--import-mode=importlib",
+ "--import-mode=importlib",
]
testpaths = ["tests"]
[tool.hatch.build.targets.sdist]
exclude = [
- "/.github",
+ "/.github",
]
+[tool.hatch.build.targets.wheel.shared-data]
+"docs/wikiget.1" = "share/man/man1/wikiget.1"
+"README.md" = "share/doc/wikiget/README.md"
+"LICENSE" = "share/doc/wikiget/LICENSE"
+
[tool.hatch.envs.default]
dependencies = [
"coverage[toml]>=6.5",
@@ -73,13 +79,15 @@ cov-report = [
"- coverage combine",
"coverage report",
]
+htmlcov = "coverage html"
cov = [
"test-cov",
"cov-report",
+ "htmlcov",
]
[[tool.hatch.envs.all.matrix]]
-python = ["3.7", "3.8", "3.9", "3.10", "3.11"]
+python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
[tool.hatch.envs.lint]
detached = true
@@ -147,8 +155,6 @@ ignore = [
"S105", "S106", "S107",
# Ignore complexity
"C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
- # FIXME: temporarily ignore usage of `print()`
- "T201",
]
unfixable = [
# Don't touch unused imports
@@ -182,5 +188,9 @@ exclude_lines = [
]
[[tool.mypy.overrides]]
-module = ["mwclient"]
+module = [
+ "mwclient",
+ "mwclient.image",
+ "pytest",
+]
ignore_missing_imports = true
diff --git a/setup.py b/setup.py
index 6068493..a73e48c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,3 +1,20 @@
+# wikiget - CLI tool for downloading files from Wikimedia sites
+# Copyright (C) 2023 Cody Logan
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Wikiget is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Wikiget is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Wikiget. If not, see <https://www.gnu.org/licenses/>.
+
from setuptools import setup
setup()
diff --git a/src/wikiget/__init__.py b/src/wikiget/__init__.py
index b68b0ec..3946868 100644
--- a/src/wikiget/__init__.py
+++ b/src/wikiget/__init__.py
@@ -1,5 +1,5 @@
# wikiget - CLI tool for downloading files from Wikimedia sites
-# Copyright (C) 2018, 2019, 2020 Cody Logan and contributors
+# Copyright (C) 2018-2023 Cody Logan and contributors
# SPDX-License-Identifier: GPL-3.0-or-later
#
# Wikiget is free software: you can redistribute it and/or modify
@@ -24,8 +24,9 @@ BLOCKSIZE = 65536
CHUNKSIZE = 1024
DEFAULT_SITE = "commons.wikimedia.org"
DEFAULT_PATH = "/w/"
-USER_AGENT = "wikiget/{} (https://github.com/clpo13/wikiget) mwclient/{}".format(
- wikiget_version, mwclient_version
+USER_AGENT = (
+ f"wikiget/{wikiget_version} (https://github.com/clpo13/wikiget) "
+ f"mwclient/{mwclient_version}"
)
STD_VERBOSE = 1
VERY_VERBOSE = 2
diff --git a/src/wikiget/dl.py b/src/wikiget/dl.py
index 949f09e..5b5b43b 100644
--- a/src/wikiget/dl.py
+++ b/src/wikiget/dl.py
@@ -1,5 +1,5 @@
# wikiget - CLI tool for downloading files from Wikimedia sites
-# Copyright (C) 2018-2021 Cody Logan and contributors
+# Copyright (C) 2018-2023 Cody Logan and contributors
# SPDX-License-Identifier: GPL-3.0-or-later
#
# Wikiget is free software: you can redistribute it and/or modify
@@ -15,125 +15,154 @@
# You should have received a copy of the GNU General Public License
# along with Wikiget. If not, see <https://www.gnu.org/licenses/>.
+import logging
import os
import sys
-from urllib.parse import unquote, urlparse
+from argparse import Namespace
+from concurrent.futures import ThreadPoolExecutor
from mwclient import APIError, InvalidResponse, LoginError, Site
+from mwclient.image import Image
from requests import ConnectionError, HTTPError
from tqdm import tqdm
import wikiget
-from wikiget.validations import valid_file, verify_hash
+from wikiget.exceptions import ParseError
+from wikiget.file import File
+from wikiget.logging import FileLogAdapter
+from wikiget.parse import get_dest
+from wikiget.validations import verify_hash
-def download(dl, args):
- url = urlparse(dl)
-
- if url.netloc:
- filename = url.path
- site_name = url.netloc
- if args.site is not wikiget.DEFAULT_SITE and not args.quiet:
- # this will work even if the user specifies 'commons.wikimedia.org'
- print("Warning: target is a URL, ignoring site specified with --site")
- else:
- filename = dl
- site_name = args.site
-
- file_match = valid_file(filename)
-
- # check if this is a valid file
- if file_match and file_match.group(1):
- # has File:/Image: prefix and extension
- filename = file_match.group(2)
- else:
- # no file extension and/or prefix, probably an article
- print(f"Could not parse input '{filename}' as a file. ")
- sys.exit(1)
-
- filename = unquote(filename) # remove URL encoding for special characters
-
- dest = args.output or filename
-
- if args.verbose >= wikiget.VERY_VERBOSE:
- print(f"User agent: {wikiget.USER_AGENT}")
-
+def query_api(filename: str, site_name: str, args: Namespace) -> Image:
# connect to site and identify ourselves
- if args.verbose >= wikiget.STD_VERBOSE:
- print(f"Site name: {site_name}")
+ logging.info(f"Connecting to {site_name}")
try:
site = Site(site_name, path=args.path, clients_useragent=wikiget.USER_AGENT)
if args.username and args.password:
site.login(args.username, args.password)
except ConnectionError as e:
- # usually this means there is no such site, or there's no network
- # connection, though it could be a certificate problem
- print("Error: couldn't connect to specified site.")
- if args.verbose >= wikiget.VERY_VERBOSE:
- print("Full error message:")
- print(e)
- sys.exit(1)
+ # usually this means there is no such site, or there's no network connection,
+ # though it could be a certificate problem
+ logging.error("Could not connect to specified site")
+ logging.debug(e)
+ raise
except HTTPError as e:
# most likely a 403 forbidden or 404 not found error for api.php
- print(
- "Error: couldn't find the specified wiki's api.php. "
- "Check the value of --path."
+ logging.error(
+ "Could not find the specified wiki's api.php. Check the value of --path."
)
- if args.verbose >= wikiget.VERY_VERBOSE:
- print("Full error message:")
- print(e)
- sys.exit(1)
+ logging.debug(e)
+ raise
except (InvalidResponse, LoginError) as e:
- # InvalidResponse: site exists, but we couldn't communicate with the
- # API endpoint for some reason other than an HTTP error.
+ # InvalidResponse: site exists, but we couldn't communicate with the API
+ # endpoint for some reason other than an HTTP error.
# LoginError: missing or invalid credentials
- print(e)
- sys.exit(1)
+ logging.error(e)
+ raise
# get info about the target file
try:
- file = site.images[filename]
+ image = site.images[filename]
except APIError as e:
- # an API error at this point likely means access is denied,
- # which could happen with a private wiki
- print(
- "Error: access denied. Try providing credentials with "
- "--username and --password."
+ # an API error at this point likely means access is denied, which could happen
+ # with a private wiki
+ logging.error(
+ "Access denied. Try providing credentials with --username and --password."
)
- if args.verbose >= wikiget.VERY_VERBOSE:
- print("Full error message:")
- for i in e.args:
- print(i)
- sys.exit(1)
+ for i in e.args:
+ logging.debug(i)
+ raise
- if file.imageinfo != {}:
- # file exists either locally or at a common repository,
- # like Wikimedia Commons
+ return image
+
+
+def prep_download(dl: str, args: Namespace) -> File:
+ file = get_dest(dl, args)
+ file.image = query_api(file.name, file.site, args)
+ return file
+
+
+def batch_download(args: Namespace) -> int:
+ input_file = args.FILE
+ dl_list = {}
+ errors = 0
+
+ logging.info(f"Using batch file '{input_file}'.")
+
+ try:
+ fd = open(input_file)
+ except OSError as e:
+ logging.error("File could not be read. The following error was encountered:")
+ logging.error(e)
+ sys.exit(1)
+ else:
+ with fd:
+ # read the file into memory and process each line as we go
+ for line_num, line in enumerate(fd, start=1):
+ line_s = line.strip()
+ # ignore blank lines and lines starting with "#" (for comments)
+ if line_s and not line_s.startswith("#"):
+ dl_list[line_num] = line_s
+
+ # TODO: validate file contents before download process starts
+ with ThreadPoolExecutor(max_workers=args.threads) as executor:
+ futures = []
+ for line_num, line in dl_list.items():
+ # keep track of batch file line numbers for debugging/logging purposes
+ logging.info(f"Processing '{line}' at line {line_num}")
+ try:
+ file = prep_download(line, args)
+ except ParseError as e:
+ logging.warning(f"{e} (line {line_num})")
+ errors += 1
+ continue
+ except (ConnectionError, HTTPError, InvalidResponse, LoginError, APIError):
+ logging.warning(
+ f"Unable to download '{line}' (line {line_num}) due to an error"
+ )
+ errors += 1
+ continue
+ future = executor.submit(download, file, args)
+ futures.append(future)
+ # wait for downloads to finish
+ for future in futures:
+ errors += future.result()
+ return errors
+
+
+def download(f: File, args: Namespace) -> int:
+ file = f.image
+ filename = f.name
+ dest = f.dest
+ site = file.site
+
+ errors = 0
+
+ logger = logging.getLogger("")
+ adapter = FileLogAdapter(logger, {"filename": filename})
+
+ if file.exists:
+ # file exists either locally or at a common repository, like Wikimedia Commons
file_url = file.imageinfo["url"]
file_size = file.imageinfo["size"]
file_sha1 = file.imageinfo["sha1"]
- if args.verbose >= wikiget.STD_VERBOSE:
- print(
- f"Info: downloading '{filename}' "
- f"({file_size} bytes) from {site.host}",
- end="",
- )
- if args.output:
- print(f" to '{dest}'")
- else:
- print("\n", end="")
- print(f"Info: {file_url}")
+ filename_log = f"Downloading '{filename}' ({file_size} bytes) from {site.host}"
+ if args.output:
+ filename_log += f" to '{dest}'"
+ adapter.info(filename_log)
+ adapter.info(f"{file_url}")
if os.path.isfile(dest) and not args.force:
- print(f"File '{dest}' already exists, skipping download (use -f to ignore)")
+ adapter.warning("File already exists, skipping download (use -f to force)")
+ errors += 1
else:
try:
fd = open(dest, "wb")
except OSError as e:
- print("File could not be written. The following error was encountered:")
- print(e)
- sys.exit(1)
+ adapter.error(f"File could not be written. {e}")
+ errors += 1
else:
# download the file(s)
if args.verbose >= wikiget.STD_VERBOSE:
@@ -154,21 +183,25 @@ def download(dl, args):
fd.write(chunk)
progress_bar.update(len(chunk))
- # verify file integrity and optionally print details
+ # verify file integrity and log details
dl_sha1 = verify_hash(dest)
- if args.verbose >= wikiget.STD_VERBOSE:
- print(f"Info: downloaded file SHA1 is {dl_sha1}")
- print(f"Info: server file SHA1 is {file_sha1}")
+ adapter.info(f"Remote file SHA1 is {file_sha1}")
+ adapter.info(f"Local file SHA1 is {dl_sha1}")
if dl_sha1 == file_sha1:
- if args.verbose >= wikiget.STD_VERBOSE:
- print("Info: hashes match!")
+ adapter.info("Hashes match!")
# at this point, we've successfully downloaded the file
+ success_log = f"'{filename}' downloaded"
+ if args.output:
+ success_log += f" to '{dest}'"
+ adapter.info(success_log)
else:
- print("Error: hash mismatch! Downloaded file may be corrupt.")
- sys.exit(1)
+ adapter.error("Hash mismatch! Downloaded file may be corrupt.")
+ errors += 1
else:
# no file information returned
- print(f"Target '{filename}' does not appear to be a valid file.")
- sys.exit(1)
+ adapter.warning("Target does not appear to be a valid file")
+ errors += 1
+
+ return errors
diff --git a/src/wikiget/exceptions.py b/src/wikiget/exceptions.py
new file mode 100644
index 0000000..94ed6b2
--- /dev/null
+++ b/src/wikiget/exceptions.py
@@ -0,0 +1,20 @@
+# wikiget - CLI tool for downloading files from Wikimedia sites
+# Copyright (C) 2023 Cody Logan
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Wikiget is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Wikiget is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Wikiget. If not, see <https://www.gnu.org/licenses/>.
+
+
+class ParseError(Exception):
+ pass
diff --git a/src/wikiget/file.py b/src/wikiget/file.py
new file mode 100644
index 0000000..b890e63
--- /dev/null
+++ b/src/wikiget/file.py
@@ -0,0 +1,39 @@
+# wikiget - CLI tool for downloading files from Wikimedia sites
+# Copyright (C) 2023 Cody Logan
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Wikiget is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Wikiget is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Wikiget. If not, see <https://www.gnu.org/licenses/>.
+
+from mwclient.image import Image
+
+from wikiget import DEFAULT_SITE
+
+
+class File:
+ def __init__(self, name: str, dest: str = "", site: str = "") -> None:
+ """
+ Initializes a new file with the specified name and an optional destination name.
+
+ :param name: name of the file
+ :type name: str
+ :param dest: destination of the file, if different from the name; if not
+ specified, defaults to the name
+ :type dest: str, optional
+ :param site: name of the site hosting the file; if not specified, defaults to
+ the global default site
+ """
+ self.image: Image = None
+ self.name = name
+ self.dest = dest if dest else name
+ self.site = site if site else DEFAULT_SITE
diff --git a/src/wikiget/logging.py b/src/wikiget/logging.py
new file mode 100644
index 0000000..87b917c
--- /dev/null
+++ b/src/wikiget/logging.py
@@ -0,0 +1,58 @@
+# wikiget - CLI tool for downloading files from Wikimedia sites
+# Copyright (C) 2023 Cody Logan
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Wikiget is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Wikiget is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Wikiget. If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+from argparse import Namespace
+
+import wikiget
+
+
+class FileLogAdapter(logging.LoggerAdapter):
+ def process(self, msg, kwargs):
+ return f"[{self.extra['filename']}] {msg}", kwargs
+
+
+def configure_logging(args: Namespace) -> None:
+ loglevel = logging.WARNING
+ if args.verbose >= wikiget.VERY_VERBOSE:
+ # this includes API and library messages
+ loglevel = logging.DEBUG
+ elif args.verbose >= wikiget.STD_VERBOSE:
+ loglevel = logging.INFO
+ elif args.quiet:
+ loglevel = logging.ERROR
+
+ # configure logging:
+ # console log level is set via -v, -vv, and -q options;
+ # file log level is always debug (TODO: make this user configurable)
+ base_format = "%(message)s"
+ log_format = "[%(levelname)s] " + base_format
+ if args.logfile:
+ # log to console and file
+ logging.basicConfig(
+ level=logging.DEBUG,
+ format="%(asctime)s [%(levelname)-7s] " + base_format,
+ filename=args.logfile,
+ )
+
+ console = logging.StreamHandler()
+ console.setLevel(loglevel)
+ console.setFormatter(logging.Formatter(log_format))
+ logging.getLogger("").addHandler(console)
+ else:
+ # log only to console
+ logging.basicConfig(level=loglevel, format=log_format)
diff --git a/src/wikiget/parse.py b/src/wikiget/parse.py
new file mode 100644
index 0000000..fe3fe43
--- /dev/null
+++ b/src/wikiget/parse.py
@@ -0,0 +1,59 @@
+# wikiget - CLI tool for downloading files from Wikimedia sites
+# Copyright (C) 2023 Cody Logan
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Wikiget is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Wikiget is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Wikiget. If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+from argparse import Namespace
+from urllib.parse import unquote, urlparse
+
+import wikiget
+from wikiget.exceptions import ParseError
+from wikiget.file import File
+from wikiget.validations import valid_file
+
+
+def get_dest(dl: str, args: Namespace) -> File:
+ url = urlparse(dl)
+
+ if url.netloc:
+ filename = url.path
+ site_name = url.netloc
+ if args.site is not wikiget.DEFAULT_SITE:
+ # this will work even if the user specifies 'commons.wikimedia.org' since
+ # we're comparing objects instead of values (is not vs. !=)
+ logging.warning("Target is a URL, ignoring site specified with --site")
+ else:
+ filename = dl
+ site_name = args.site
+
+ file_match = valid_file(filename)
+
+ # check if this is a valid file
+ if file_match and file_match.group(1):
+ # has File:/Image: prefix and extension
+ filename = file_match.group(2)
+ else:
+ # no file extension and/or prefix, probably an article
+ msg = f"Could not parse input '{filename}' as a file"
+ raise ParseError(msg)
+
+ filename = unquote(filename) # remove URL encoding for special characters
+
+ dest = args.output or filename
+
+ file = File(filename, dest, site_name)
+
+ return file
diff --git a/src/wikiget/validations.py b/src/wikiget/validations.py
index dc70df4..c9e7bcf 100644
--- a/src/wikiget/validations.py
+++ b/src/wikiget/validations.py
@@ -1,5 +1,5 @@
# wikiget - CLI tool for downloading files from Wikimedia sites
-# Copyright (C) 2018, 2019, 2020 Cody Logan
+# Copyright (C) 2018-2023 Cody Logan
# SPDX-License-Identifier: GPL-3.0-or-later
#
# Wikiget is free software: you can redistribute it and/or modify
@@ -17,17 +17,21 @@
import hashlib
import re
+from typing import Optional
from wikiget import BLOCKSIZE
-def valid_file(search_string):
+def valid_file(search_string: str) -> Optional[re.Match]:
"""
- Determines if the given string contains a valid file name, defined as a
- string ending with a '.' and at least one character, beginning with 'File:'
- or 'Image:', the standard file prefixes in MediaWiki.
+ Determines if the given string contains a valid file name, defined as a string
+ ending with a '.' and at least one character, beginning with 'File:' or 'Image:',
+ the standard file prefixes in MediaWiki.
+
:param search_string: string to validate
+ :type search_string: str
:returns: a regex Match object if there's a match or None otherwise
+ :rtype: re.Match
"""
# second group could also restrict to file extensions with three or more
# letters with ([^/\r\n\t\f\v]+\.\w{3,})
@@ -35,25 +39,30 @@ def valid_file(search_string):
return file_regex.search(search_string)
-def valid_site(search_string):
+def valid_site(search_string: str) -> Optional[re.Match]:
"""
- Determines if the given string contains a valid site name, defined as a
- string ending with 'wikipedia.org' or 'wikimedia.org'. This covers all
- subdomains of those domains. Eventually, it should be possible to support
- any MediaWiki site, regardless of domain name.
+ Determines if the given string contains a valid site name, defined as a string
+ ending with 'wikipedia.org' or 'wikimedia.org'. This covers all subdomains of those
+ domains. Eventually, it should be possible to support any MediaWiki site, regardless
+ of domain name.
+
:param search_string: string to validate
+ :type search_string: str
:returns: a regex Match object if there's a match or None otherwise
+ :rtype: re.Match
"""
site_regex = re.compile(r"wiki[mp]edia\.org$", re.I)
return site_regex.search(search_string)
-def verify_hash(filename):
+def verify_hash(filename: str) -> str:
"""
- Calculates the SHA1 hash of the given file for comparison with a known
- value.
+ Calculates the SHA1 hash of the given file for comparison with a known value.
+
:param filename: name of the file to calculate a hash for
+ :type filename: str
:return: hash digest
+ :rtype: str
"""
hasher = hashlib.sha1() # noqa: S324
with open(filename, "rb") as dl:
diff --git a/src/wikiget/version.py b/src/wikiget/version.py
index dd9b22c..34dabb7 100644
--- a/src/wikiget/version.py
+++ b/src/wikiget/version.py
@@ -1 +1,18 @@
+# wikiget - CLI tool for downloading files from Wikimedia sites
+# Copyright (C) 2018-2023 Cody Logan and contributors
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Wikiget is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Wikiget is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Wikiget. If not, see <https://www.gnu.org/licenses/>.
+
__version__ = "0.5.1"
diff --git a/src/wikiget/wikiget.py b/src/wikiget/wikiget.py
index ba36766..e64d00e 100644
--- a/src/wikiget/wikiget.py
+++ b/src/wikiget/wikiget.py
@@ -1,5 +1,5 @@
# wikiget - CLI tool for downloading files from Wikimedia sites
-# Copyright (C) 2018-2021 Cody Logan and contributors
+# Copyright (C) 2018-2023 Cody Logan and contributors
# SPDX-License-Identifier: GPL-3.0-or-later
#
# Wikiget is free software: you can redistribute it and/or modify
@@ -19,38 +19,33 @@ import argparse
import logging
import sys
-import wikiget
-from wikiget.dl import download
+from mwclient import APIError, InvalidResponse, LoginError
+from requests import ConnectionError, HTTPError
+import wikiget
+from wikiget.dl import batch_download, download, prep_download
+from wikiget.exceptions import ParseError
+from wikiget.logging import configure_logging
-def main():
- """
- Main entry point for console script. Automatically compiled by setuptools
- when installed with `pip install` or `python setup.py install`.
- """
+def construct_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="""
- A tool for downloading files from
- MediaWiki sites using the file name or
+ A tool for downloading files from MediaWiki sites using the file name or
description page URL
""",
epilog="""
- Copyright (C) 2018-2023 Cody Logan
- and contributors.
- License GPLv3+: GNU GPL version 3 or later
- <http://www.gnu.org/licenses/gpl.html>.
- This is free software; you are free to
- change and redistribute it under certain
- conditions. There is NO WARRANTY, to the
- extent permitted by law.
+ Copyright (C) 2018-2023 Cody Logan and contributors. License GPLv3+: GNU GPL
+ version 3 or later <http://www.gnu.org/licenses/gpl.html>. This is free
+ software; you are free to change and redistribute it under certain conditions.
+ There is NO WARRANTY, to the extent permitted by law.
""",
)
parser.add_argument(
"FILE",
help="""
- name of the file to download with the File:
- prefix, or the URL of its file description page
+ name of the file to download with the File: prefix, or the URL of its file
+ description page
""",
)
parser.add_argument(
@@ -80,52 +75,76 @@ def main():
help="MediaWiki site to download from (default: %(default)s)",
)
parser.add_argument(
- "-p",
+ "-P",
"--path",
default=wikiget.DEFAULT_PATH,
help="MediaWiki site path, where api.php is located (default: %(default)s)",
)
parser.add_argument(
- "--username", default="", help="MediaWiki site username, for private wikis"
+ "-u",
+ "--username",
+ default="",
+ help="MediaWiki site username, for private wikis",
)
parser.add_argument(
- "--password", default="", help="MediaWiki site password, for private wikis"
+ "-p",
+ "--password",
+ default="",
+ help="MediaWiki site password, for private wikis",
)
output_options = parser.add_mutually_exclusive_group()
output_options.add_argument("-o", "--output", help="write download to OUTPUT")
output_options.add_argument(
"-a",
"--batch",
- help="treat FILE as a textfile containing "
- "multiple files to download, one URL or "
- "filename per line",
+ help="treat FILE as a textfile containing multiple files to download, one URL "
+ "or filename per line",
action="store_true",
)
+ parser.add_argument(
+ "-l", "--logfile", default="", help="save log output to LOGFILE"
+ )
+ parser.add_argument(
+ "-j",
+ "--threads",
+ default=1,
+ help="number of parallel downloads to attempt in batch mode",
+ type=int,
+ )
+ return parser
+
+
+def main() -> None:
+ # setup our environment
+ parser = construct_parser()
args = parser.parse_args()
+ configure_logging(args)
- # print API and debug messages in verbose mode
- if args.verbose >= wikiget.VERY_VERBOSE:
- logging.basicConfig(level=logging.DEBUG)
- elif args.verbose >= wikiget.STD_VERBOSE:
- logging.basicConfig(level=logging.WARNING)
+ # log events are appended to the file if it already exists, so note the start of a
+ # new download session
+ logging.info(f"Starting download session using wikiget {wikiget.wikiget_version}")
+ logging.debug(f"User agent: {wikiget.USER_AGENT}")
if args.batch:
# batch download mode
- input_file = args.FILE
- if args.verbose >= wikiget.STD_VERBOSE:
- print(f"Info: using batch file '{input_file}'")
- try:
- fd = open(input_file)
- except OSError as e:
- print("File could not be read. The following error was encountered:")
- print(e)
+ errors = batch_download(args)
+ if errors:
+ # return non-zero exit code if any problems were encountered, even if some
+ # downloads completed successfully
+ logging.warning(
+ f"{errors} problem{'s'[:errors^1]} encountered during batch processing"
+ )
sys.exit(1)
- else:
- with fd:
- for _, line in enumerate(fd):
- download(line.strip(), args)
else:
# single download mode
- dl = args.FILE
- download(dl, args)
+ try:
+ file = prep_download(args.FILE, args)
+ except ParseError as e:
+ logging.error(e)
+ sys.exit(1)
+ except (ConnectionError, HTTPError, InvalidResponse, LoginError, APIError):
+ sys.exit(1)
+ errors = download(file, args)
+ if errors:
+ sys.exit(1)
diff --git a/tests/test_dl.py b/tests/test_dl.py
new file mode 100644
index 0000000..fc68733
--- /dev/null
+++ b/tests/test_dl.py
@@ -0,0 +1,46 @@
+# wikiget - CLI tool for downloading files from Wikimedia sites
+# Copyright (C) 2023 Cody Logan
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Wikiget is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Wikiget is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Wikiget. If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+
+import pytest
+
+from wikiget import USER_AGENT
+from wikiget.dl import prep_download, query_api
+from wikiget.wikiget import construct_parser
+
+
+# TODO: don't hit the actual API when doing tests
+@pytest.mark.skip
+class TestQueryApi:
+ parser = construct_parser()
+
+ def test_query_api(self, caplog):
+ caplog.set_level(logging.DEBUG)
+ args = self.parser.parse_args(["File:Example.jpg"])
+ file, site = query_api("Example.jpg", "commons.wikimedia.org", args)
+ assert USER_AGENT in caplog.text
+
+
+@pytest.mark.skip
+class TestPrepDownload:
+ parser = construct_parser()
+
+ def test_prep_download(self):
+ args = self.parser.parse_args(["File:Example.jpg"])
+ file = prep_download(args.FILE, args)
+ assert file is not None
diff --git a/tests/test_file_class.py b/tests/test_file_class.py
new file mode 100644
index 0000000..dd30207
--- /dev/null
+++ b/tests/test_file_class.py
@@ -0,0 +1,37 @@
+# wikiget - CLI tool for downloading files from Wikimedia sites
+# Copyright (C) 2023 Cody Logan
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Wikiget is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Wikiget is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Wikiget. If not, see <https://www.gnu.org/licenses/>.
+
+from wikiget import DEFAULT_SITE
+from wikiget.file import File
+
+
+def test_file_with_name_only():
+ file = File("foobar.jpg")
+ assert file.name == "foobar.jpg"
+ assert file.dest == file.name
+ assert file.site == DEFAULT_SITE
+
+
+def test_file_with_name_and_dest():
+ file = File("foobar.jpg", dest="bazqux.jpg")
+ assert file.dest == "bazqux.jpg"
+ assert file.dest != file.name
+
+
+def test_file_with_name_and_site():
+ file = File("foobar.jpg", site="en.wikipedia.org")
+ assert file.site == "en.wikipedia.org"
diff --git a/tests/test_parse.py b/tests/test_parse.py
new file mode 100644
index 0000000..757b361
--- /dev/null
+++ b/tests/test_parse.py
@@ -0,0 +1,60 @@
+# wikiget - CLI tool for downloading files from Wikimedia sites
+# Copyright (C) 2023 Cody Logan
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Wikiget is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Wikiget is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Wikiget. If not, see <https://www.gnu.org/licenses/>.
+
+import pytest
+
+from wikiget.exceptions import ParseError
+from wikiget.parse import get_dest
+from wikiget.wikiget import construct_parser
+
+
+class TestGetDest:
+ parser = construct_parser()
+
+ def test_get_dest_with_filename(self):
+ args = self.parser.parse_args(["File:Example.jpg"])
+ file = get_dest(args.FILE, args)
+ assert file.name == "Example.jpg"
+ assert file.dest == "Example.jpg"
+ assert file.site == "commons.wikimedia.org"
+
+ def test_get_dest_with_url(self):
+ args = self.parser.parse_args(
+ [
+ "https://en.wikipedia.org/wiki/File:Example.jpg",
+ ]
+ )
+ file = get_dest(args.FILE, args)
+ assert file.name == "Example.jpg"
+ assert file.dest == "Example.jpg"
+ assert file.site == "en.wikipedia.org"
+
+ def test_get_dest_with_bad_filename(self):
+ args = self.parser.parse_args(["Example.jpg"])
+ with pytest.raises(ParseError):
+ _ = get_dest(args.FILE, args)
+
+ def test_get_dest_with_different_site(self, caplog: pytest.LogCaptureFixture):
+ args = self.parser.parse_args(
+ [
+ "https://commons.wikimedia.org/wiki/File:Example.jpg",
+ "--site",
+ "commons.wikimedia.org",
+ ]
+ )
+ _ = get_dest(args.FILE, args)
+ assert "Target is a URL, ignoring site specified with --site" in caplog.text
diff --git a/tests/test_validations.py b/tests/test_validations.py
index 1abd96a..9d70f6e 100644
--- a/tests/test_validations.py
+++ b/tests/test_validations.py
@@ -1,5 +1,5 @@
# wikiget - CLI tool for downloading files from Wikimedia sites
-# Copyright (C) 2018-2021 Cody Logan
+# Copyright (C) 2018-2023 Cody Logan
# SPDX-License-Identifier: GPL-3.0-or-later
#
# Wikiget is free software: you can redistribute it and/or modify