diff options
| -rw-r--r-- | .github/workflows/python.yml | 20 | ||||
| -rw-r--r-- | .gitignore | 6 | ||||
| -rw-r--r-- | MANIFEST.in | 5 | ||||
| -rw-r--r-- | README.md | 112 | ||||
| -rw-r--r-- | docs/Makefile | 6 | ||||
| -rw-r--r-- | docs/wikiget.1 | 123 | ||||
| -rw-r--r-- | docs/wikiget.1.md | 116 | ||||
| -rw-r--r-- | pyproject.toml | 68 | ||||
| -rw-r--r-- | setup.py | 17 | ||||
| -rw-r--r-- | src/wikiget/__init__.py | 7 | ||||
| -rw-r--r-- | src/wikiget/dl.py | 219 | ||||
| -rw-r--r-- | src/wikiget/exceptions.py | 20 | ||||
| -rw-r--r-- | src/wikiget/file.py | 39 | ||||
| -rw-r--r-- | src/wikiget/logging.py | 58 | ||||
| -rw-r--r-- | src/wikiget/parse.py | 59 | ||||
| -rw-r--r-- | src/wikiget/validations.py | 35 | ||||
| -rw-r--r-- | src/wikiget/version.py | 17 | ||||
| -rw-r--r-- | src/wikiget/wikiget.py | 109 | ||||
| -rw-r--r-- | tests/test_dl.py | 46 | ||||
| -rw-r--r-- | tests/test_file_class.py | 37 | ||||
| -rw-r--r-- | tests/test_parse.py | 60 | ||||
| -rw-r--r-- | tests/test_validations.py | 2 |
22 files changed, 928 insertions, 253 deletions
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index bef2670..7f44810 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -5,9 +5,9 @@ name: Python package on: push: - branches: [ "master" ] + branches: [ "dev" ] pull_request: - branches: [ "master" ] + branches: [ "master", "dev" ] jobs: build: @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 @@ -27,12 +27,20 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel - python -m pip install flake8 pytest pytest-cov + python -m pip install flake8 pytest "coverage[toml]" python -m pip install . - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names - flake8 . --count --show-source --statistics + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - name: Test with pytest run: | - pytest + coverage run -m pytest + - name: Convert coverage file to XML + run: | + coverage combine + coverage xml + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} @@ -107,3 +107,9 @@ venv.bak/ .vs/ .vscode/ .idea/ + +# downloaded images, batch test files +*.jpg +*.jpeg +*.png +batch.txt
\ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 7592f0e..429c6c8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ -graft wikiget -graft test +graft src +graft tests +graft docs global-exclude *.py[cod] @@ -3,47 +3,45 @@ [](https://github.com/clpo13/wikiget/actions/workflows/python.yml) [](https://badge.fury.io/py/wikiget) -Something like wget for downloading a file from MediaWiki sites (like Wikipedia -or Wikimedia Commons) using only the file name or the URL of its description -page. +Something like wget for downloading a file from MediaWiki sites (like Wikipedia or Wikimedia Commons) using only the +file name or the URL of its description page. -Requires Python 3.7+. Get it with `pip install --user wikiget` or `pipx install wikiget`. +Requires Python 3.7+ and pip. Get it with `pip install wikiget` or `pipx install wikiget`. For the latest features, at +the risk of bugs and undocumented behavior, you can try the dev branch: +`pip install https://github.com/clpo13/wikiget/archive/refs/heads/dev.zip` ## Usage -`wikiget [-h] [-V] [-q | -v] [-f] [-s SITE] [-p PATH] [--username USERNAME] -[--password PASSWORD] [-o OUTPUT | -a] FILE` - -If `FILE` is in the form `File:Example.jpg` or `Image:Example.jpg`, it will be -fetched from the default site, which is "commons.wikimedia.org". If it's the -fully-qualified URL of a file description page, like -`https://en.wikipedia.org/wiki/File:Example.jpg`, the file is fetched from the -specified site, in this case "en.wikipedia.org". Full URLs may contain -characters your shell interprets differently, so you can either escape those -characters with a backslash `\` or surround the entire URL with single `'` or -double `"` quotes. Use of a fully-qualified URL like this may require setting -the `--path` flag (see next paragraph). - -The site can also be specified with the `--site` flag, though this will not have -any effect if the full URL is given. Non-Wikimedia sites should work, but you -may need to specify the wiki's script path with `--path` (where `index.php` and -`api.php` live; on Wikimedia sites it's `/w/`, but other sites may use `/` or -something else entirely). Private wikis (those requiring login even for read -access) are also supported with the use of the `--username` and `--password` -flags. - -More detailed information, such as the site used and full URL of the file, can -be displayed with `-v` or `--verbose`. Use `-vv` to display even more detail. -`-q` can be used to silence warnings. - -By default, the program won't overwrite existing files with the same name as the -target, but this can be forced with `-f` or `--force`. Additionally, the file -can be downloaded to a different name with `-o`. - -Files can be batch downloaded with the `-a` or `--batch` flag. In this mode, -`FILE` will be treated as an input file containing multiple files to download, -one filename or URL per line. If an error is encountered, execution stops -immediately and the offending filename is printed. +`wikiget [-h] [-V] [-q | -v] [-f] [-s SITE] [-P PATH] [-u USERNAME] [-p PASSWORD] [-o OUTPUT | -a] [-l LOGFILE] [-j THREADS] FILE` + +The only required parameter is `FILE`, which is the file you want to download. It can either be the name of the file on +the wiki, including the namespace prefix, or a link to the file description page. If `FILE` is in the form +`File:Example.jpg` or `Image:Example.jpg`, it will be fetched from the default site, which is "commons.wikimedia.org". +If it's the fully-qualified URL of a file description page, like `https://en.wikipedia.org/wiki/File:Example.jpg`, the +file is fetched from the site in the URL, in this case "en.wikipedia.org". Note: full URLs may contain characters your +shell interprets differently, so you can either escape those characters with a backslash `\` or surround the entire URL +with single `'` or double `"` quotes. Use of a fully-qualified URL like this may require setting the `--path` flag (see +next paragraph). + +The site can also be specified with the `--site` flag, though this will not have any effect if the full URL is given. +Non-Wikimedia sites should work, but you may need to specify the wiki's script path with `--path` (where `index.php` and +`api.php` live; on Wikimedia sites it's `/w/`, but other sites may use `/` or something else entirely). Private wikis +(those requiring login even for read access) are also supported with the use of the `--username` and `--password` flags. + +More detailed information, such as the site used and full URL of the file, can be displayed with `-v` or `--verbose`. +Use `-vv` to display even more detail, mainly debugging information or API messages. `-q` can be used to silence +warnings. A logfile can be specified with `-l` or `--logfile`. If this option is present, the logfile will contain the +same information as `-v` along with timestamps. New log entries will be appended to an existing logfile. + +By default, the program won't overwrite existing files with the same name as the target, but this can be forced with +`-f` or `--force`. Additionally, the file can be downloaded to a different name with `-o`. + +Files can be batch downloaded with the `-a` or `--batch` flag. In this mode, `FILE` will be treated as an input file +containing multiple files to download, one filename or URL per line. Blank lines and lines starting with "#" are +ignored. If an error is encountered, execution stops immediately and the offending filename is printed. For large +batches, the process can be sped up by downloading files in parallel. The number of parallel downloads can be set with +`-j`. For instance, with `-a -j4`, wikiget will download four files at once. Without `-j` or with `-j` by itself without +a number, wikiget will download the files one at a time. ### Example usage @@ -62,17 +60,15 @@ wikiget https://en.wikipedia.org/wiki/File:Example.jpg -o test.jpg ## Contributing -Pull requests or bug reports are more than welcome. +Pull requests, bug reports, or feature requests are more than welcome. It's recommended that you use a [virtual environment manager](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/) -like venv or [virtualenv](https://virtualenv.pypa.io/en/latest/) to create an -isolated environment in which to install this package's dependencies so as not -to clutter your system Python environment: +like venv or [virtualenv](https://virtualenv.pypa.io/en/latest/) to create an isolated environment in which to install +this package's dependencies so as not to clutter your system Python environment: ```bash -# if you plan on submitting pull requests, fork the repo on GitHub -# and clone that instead +# if you plan on submitting pull requests, fork the repo on GitHub and clone that instead git clone https://github.com/clpo13/wikiget cd wikiget @@ -92,29 +88,23 @@ source venv/bin/activate ``` Then run `pip install -e .` to invoke an -["editable" install](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs), -meaning any changes made to the source will be reflected immediately in the executable -script. Unit tests can be run with `pytest` (make sure to run `pip install pytest` -in the virtual environment first.) +["editable" install](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs) +meaning any changes made to the source will be reflected immediately in the executable script. Unit tests can be run +with `pytest` (make sure to run `pip install pytest` in the virtual environment first.) -Alternatively, using [Hatch](https://hatch.pypa.io/latest/), simply clone the repository -and run `hatch run test` to create the environment and run pytest. Also try `hatch shell` -or `hatch run wikiget --help`. +Alternatively, using [Hatch](https://hatch.pypa.io/latest/), simply clone the repository and run `hatch run test` to +create the environment and run pytest. Also try `hatch shell` or `hatch run wikiget --help`. ## License Copyright (C) 2018-2023 Cody Logan and contributors -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. +This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public +License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later +version. -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. +This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -You should have received a copy of the GNU General Public License -along with this program (see [LICENSE](LICENSE)). If not, see -<https://www.gnu.org/licenses/>. +You should have received a copy of the GNU General Public License along with this program (see [LICENSE](LICENSE)). +If not, see <https://www.gnu.org/licenses/>. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..6ce62df --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,6 @@ +.PHONY: man + +man: wikiget.1 + +wikiget.1: wikiget.1.md + pandoc -s -f markdown -t man -o wikiget.1 wikiget.1.md diff --git a/docs/wikiget.1 b/docs/wikiget.1 new file mode 100644 index 0000000..fa1a33d --- /dev/null +++ b/docs/wikiget.1 @@ -0,0 +1,123 @@ +.\" Automatically generated by Pandoc 3.1.8 +.\" +.TH "WIKIGET" "1" "October 5, 2023" "Version 0.5.1" "Wikiget User Manual" +.SH NAME +wikiget - download files from MediaWiki sites +.SH SYNOPSIS +.PP +\f[B]wikiget\f[R] [\f[I]options\f[R]] \f[I]FILE\f[R] +.PD 0 +.P +.PD +\f[B]wikiget\f[R] [\f[I]options\f[R]] [-\f[B]a\f[R]|--\f[B]batch\f[R]] +\f[I]BATCHFILE\f[R] +.PD 0 +.P +.PD +\f[B]wikiget\f[R] [-\f[B]V\f[R]|--\f[B]version\f[R]] +.PD 0 +.P +.PD +\f[B]wikiget\f[R] [-\f[B]h\f[R]|--\f[B]help\f[R]] +.SH DESCRIPTION +Something like \f[B]wget\f[R](1) for downloading a file from MediaWiki +sites (like Wikipedia or Wikimedia Commons) using only the file name or +the URL of its description page. +.SH OPTIONS +.TP +\f[I]FILE\f[R] +The file to be downloaded. +If \f[I]FILE\f[R] is in the form \f[I]File:Example.jpg\f[R] or +\f[I]Image:Example.jpg\f[R], it will be fetched from the default site, +which is \[lq]commons.wikimedia.org\[rq]. +If it\[cq]s the fully-qualified URL of a file description page, like +\f[I]https://en.wikipedia.org/wiki/File:Example.jpg\f[R], the file is +fetched from the site in the URL, in this case +\[lq]en.wikipedia.org\[rq]. +.TP +\f[I]BATCHFILE\f[R] +In batch download mode (activated with -\f[B]a\f[R] or +--\f[B]batch\f[R]), this is a text file containing multiple file names +or URLs to be downloaded, one per line. +Blank lines and lines starting with \[lq]#\[rq] are ignored. +If an error is encountered during download, execution stops immediately +and the offending filename is printed. +.TP +-\f[B]s\f[R], --\f[B]site\f[R] \f[I]SITE\f[R] +MediaWiki site to download from. +Will not have any effect if the full URL is given in the \f[I]FILE\f[R] +parameter. +.TP +-\f[B]P\f[R], --\f[B]path\f[R] \f[I]PATH\f[R] +Script path for the wiki, where \[lq]index.php\[rq] and +\[lq]api.php\[rq] live. +On Wikimedia sites, it\[cq]s \[lq]/w/\[rq], the default, but other sites +may use \[lq]/\[rq] or something else entirely. +.TP +-\f[B]u\f[R], --\f[B]username\f[R] \f[I]USERNAME\f[R] +Username for private wikis that require a login even for read access. +.TP +-\f[B]p\f[R], --\f[B]password\f[R] \f[I]PASSWORD\f[R] +Password for private wikis that require a login even for read access. +.TP +-\f[B]o\f[R], --\f[B]output\f[R] \f[I]OUTPUT\f[R] +By default, the output filename is the same as the remote filename +(without the File: or Image: prefix), but this can be changed with this +option. +.TP +-\f[B]l\f[R], --\f[B]logfile\f[R] \f[I]LOGFILE\f[R] +Specify a logfile, which will contain detailed information about the +download process. +If the logfile already exists, new log information is appended to it. +.TP +-\f[B]f\f[R], --\f[B]force\f[R] +Force existing files to be overwritten. +.TP +-\f[B]a\f[R], --\f[B]batch\f[R] +If this flag is set, \f[B]wikiget\f[R] will run in batch download mode +(see \f[I]BATCHFILE\f[R]). +.TP +-\f[B]j\f[R], --\f[B]threads\f[R] +Number of parallel downloads to attempt in batch mode. +This option has no effect if -\f[B]a\f[R] is not also set. +.TP +-\f[B]v\f[R], --\f[B]verbose\f[R] +Print additional information, such as the site used and the full URL of +the file. +Additional invocations will increase the level of detail. +.TP +-\f[B]q\f[R], --\f[B]quiet\f[R] +Silence warnings and minimize printed output. +.TP +-\f[B]V\f[R], --\f[B]version\f[R] +Print the version number of the program. +.TP +-\f[B]h\f[R], --\f[B]help\f[R] +Print a brief summary of these options. +.SH EXAMPLES +.IP +.EX +wikiget File:Example.jpg +wikiget --site en.wikipedia.org File:Example.jpg +wikiget https://en.wikipedia.org/wiki/File:Example.jpg -o test.jpg +.EE +.SH BUG REPORTS +https://github.com/clpo13/wikiget/issues +.SH LICENSE +Copyright (C) 2018-2023 Cody Logan and contributors +.PP +This program is free software: you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at your +option) any later version. +.PP +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +See the GNU General Public License for more details. +.PP +You should have received a copy of the GNU General Public License along +with this program. +If not, see https://www.gnu.org/licenses/. +.SH AUTHORS +Cody Logan <clpo13@gmail.com>. diff --git a/docs/wikiget.1.md b/docs/wikiget.1.md new file mode 100644 index 0000000..fd274d5 --- /dev/null +++ b/docs/wikiget.1.md @@ -0,0 +1,116 @@ +% WIKIGET(1) Version 0.5.1 | Wikiget User Manual +% Cody Logan <clpo13@gmail.com> +% October 5, 2023 + +# NAME + +wikiget - download files from MediaWiki sites + +# SYNOPSIS + +| **wikiget** \[*options*] *FILE* +| **wikiget** \[*options*] \[\-**a**|\-\-**batch**] *BATCHFILE* +| **wikiget** \[\-**V**|\-\-**version**] +| **wikiget** \[\-**h**|\-\-**help**] + +# DESCRIPTION + +Something like **wget**(1) for downloading a file from MediaWiki sites (like Wikipedia or Wikimedia Commons) using only +the file name or the URL of its description page. + +# OPTIONS + +*FILE* + +: The file to be downloaded. If *FILE* is in the form *File:Example.jpg* or *Image:Example.jpg*, it will be fetched + from the default site, which is "commons.wikimedia.org". If it's the fully-qualified URL of a file description page, + like *https://en.wikipedia.org/wiki/File:Example.jpg*, the file is fetched from the site in the URL, in this case + "en.wikipedia.org". + +*BATCHFILE* + +: In batch download mode (activated with \-**a** or \-\-**batch**), this is a text file containing multiple file names + or URLs to be downloaded, one per line. Blank lines and lines starting with "#" are ignored. If an error is + encountered during download, execution stops immediately and the offending filename is printed. + +\-**s**, \-\-**site** *SITE* + +: MediaWiki site to download from. Will not have any effect if the full URL is given in the *FILE* parameter. + +\-**P**, \-\-**path** *PATH* + +: Script path for the wiki, where "index.php" and "api.php" live. On Wikimedia sites, it's "/w/", the default, but + other sites may use "/" or something else entirely. + +\-**u**, \-\-**username** *USERNAME* + +: Username for private wikis that require a login even for read access. + +\-**p**, \-\-**password** *PASSWORD* + +: Password for private wikis that require a login even for read access. + +\-**o**, \-\-**output** *OUTPUT* + +: By default, the output filename is the same as the remote filename (without the File: or Image: prefix), but this + can be changed with this option. + +\-**l**, \-\-**logfile** *LOGFILE* + +: Specify a logfile, which will contain detailed information about the download process. If the logfile already + exists, new log information is appended to it. + +\-**f**, \-\-**force** + +: Force existing files to be overwritten. + +\-**a**, \-\-**batch** + +: If this flag is set, **wikiget** will run in batch download mode (see *BATCHFILE*). + +\-**j**, \-\-**threads** + +: Number of parallel downloads to attempt in batch mode. This option has no effect if \-**a** is not also set. + +\-**v**, \-\-**verbose** + +: Print additional information, such as the site used and the full URL of the file. Additional invocations will + increase the level of detail. + +\-**q**, \-\-**quiet** + +: Silence warnings and minimize printed output. + +\-**V**, \-\-**version** + +: Print the version number of the program. + +\-**h**, \-\-**help** + +: Print a brief summary of these options. + +# EXAMPLES + +``` +wikiget File:Example.jpg +wikiget --site en.wikipedia.org File:Example.jpg +wikiget https://en.wikipedia.org/wiki/File:Example.jpg -o test.jpg +``` + +# BUG REPORTS + +https://github.com/clpo13/wikiget/issues + +# LICENSE + +Copyright (C) 2018-2023 Cody Logan and contributors + +This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public +License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with this program. If not, see +https://www.gnu.org/licenses/. diff --git a/pyproject.toml b/pyproject.toml index aab4b3f..11fcaad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,36 +8,37 @@ dynamic = ["version"] description = "CLI tool for downloading files from MediaWiki sites" readme = "README.md" authors = [ - {name = "Cody Logan", email = "clpo13@gmail.com"} + {name = "Cody Logan", email = "clpo13@gmail.com"} ] requires-python = ">=3.7" license = {text = "GPL-3.0-or-later"} keywords = ["commons", "mediawiki", "wikimedia", "wikipedia"] classifiers = [ - "Development Status :: 4 - Beta", - "Environment :: Console", - "Intended Audience :: End Users/Desktop", - "Operating System :: OS Independent", - "Topic :: Internet", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Multimedia", - "Topic :: Multimedia :: Graphics", - "Topic :: Multimedia :: Sound/Audio", - "Topic :: Multimedia :: Video", - "Topic :: Utilities", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: End Users/Desktop", + "Operating System :: OS Independent", + "Topic :: Internet", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Multimedia", + "Topic :: Multimedia :: Graphics", + "Topic :: Multimedia :: Sound/Audio", + "Topic :: Multimedia :: Video", + "Topic :: Utilities", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dependencies = [ - "mwclient>=0.10.0", - "requests", - "tqdm", + "mwclient>=0.10.0", + "requests", + "tqdm", ] [project.urls] @@ -52,15 +53,20 @@ path = "src/wikiget/version.py" [tool.pytest.ini_options] addopts = [ - "--import-mode=importlib", + "--import-mode=importlib", ] testpaths = ["tests"] [tool.hatch.build.targets.sdist] exclude = [ - "/.github", + "/.github", ] +[tool.hatch.build.targets.wheel.shared-data] +"docs/wikiget.1" = "share/man/man1/wikiget.1" +"README.md" = "share/doc/wikiget/README.md" +"LICENSE" = "share/doc/wikiget/LICENSE" + [tool.hatch.envs.default] dependencies = [ "coverage[toml]>=6.5", @@ -73,13 +79,15 @@ cov-report = [ "- coverage combine", "coverage report", ] +htmlcov = "coverage html" cov = [ "test-cov", "cov-report", + "htmlcov", ] [[tool.hatch.envs.all.matrix]] -python = ["3.7", "3.8", "3.9", "3.10", "3.11"] +python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] [tool.hatch.envs.lint] detached = true @@ -147,8 +155,6 @@ ignore = [ "S105", "S106", "S107", # Ignore complexity "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", - # FIXME: temporarily ignore usage of `print()` - "T201", ] unfixable = [ # Don't touch unused imports @@ -182,5 +188,9 @@ exclude_lines = [ ] [[tool.mypy.overrides]] -module = ["mwclient"] +module = [ + "mwclient", + "mwclient.image", + "pytest", +] ignore_missing_imports = true @@ -1,3 +1,20 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2023 Cody Logan +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see <https://www.gnu.org/licenses/>. + from setuptools import setup setup() diff --git a/src/wikiget/__init__.py b/src/wikiget/__init__.py index b68b0ec..3946868 100644 --- a/src/wikiget/__init__.py +++ b/src/wikiget/__init__.py @@ -1,5 +1,5 @@ # wikiget - CLI tool for downloading files from Wikimedia sites -# Copyright (C) 2018, 2019, 2020 Cody Logan and contributors +# Copyright (C) 2018-2023 Cody Logan and contributors # SPDX-License-Identifier: GPL-3.0-or-later # # Wikiget is free software: you can redistribute it and/or modify @@ -24,8 +24,9 @@ BLOCKSIZE = 65536 CHUNKSIZE = 1024 DEFAULT_SITE = "commons.wikimedia.org" DEFAULT_PATH = "/w/" -USER_AGENT = "wikiget/{} (https://github.com/clpo13/wikiget) mwclient/{}".format( - wikiget_version, mwclient_version +USER_AGENT = ( + f"wikiget/{wikiget_version} (https://github.com/clpo13/wikiget) " + f"mwclient/{mwclient_version}" ) STD_VERBOSE = 1 VERY_VERBOSE = 2 diff --git a/src/wikiget/dl.py b/src/wikiget/dl.py index 949f09e..5b5b43b 100644 --- a/src/wikiget/dl.py +++ b/src/wikiget/dl.py @@ -1,5 +1,5 @@ # wikiget - CLI tool for downloading files from Wikimedia sites -# Copyright (C) 2018-2021 Cody Logan and contributors +# Copyright (C) 2018-2023 Cody Logan and contributors # SPDX-License-Identifier: GPL-3.0-or-later # # Wikiget is free software: you can redistribute it and/or modify @@ -15,125 +15,154 @@ # You should have received a copy of the GNU General Public License # along with Wikiget. If not, see <https://www.gnu.org/licenses/>. +import logging import os import sys -from urllib.parse import unquote, urlparse +from argparse import Namespace +from concurrent.futures import ThreadPoolExecutor from mwclient import APIError, InvalidResponse, LoginError, Site +from mwclient.image import Image from requests import ConnectionError, HTTPError from tqdm import tqdm import wikiget -from wikiget.validations import valid_file, verify_hash +from wikiget.exceptions import ParseError +from wikiget.file import File +from wikiget.logging import FileLogAdapter +from wikiget.parse import get_dest +from wikiget.validations import verify_hash -def download(dl, args): - url = urlparse(dl) - - if url.netloc: - filename = url.path - site_name = url.netloc - if args.site is not wikiget.DEFAULT_SITE and not args.quiet: - # this will work even if the user specifies 'commons.wikimedia.org' - print("Warning: target is a URL, ignoring site specified with --site") - else: - filename = dl - site_name = args.site - - file_match = valid_file(filename) - - # check if this is a valid file - if file_match and file_match.group(1): - # has File:/Image: prefix and extension - filename = file_match.group(2) - else: - # no file extension and/or prefix, probably an article - print(f"Could not parse input '{filename}' as a file. ") - sys.exit(1) - - filename = unquote(filename) # remove URL encoding for special characters - - dest = args.output or filename - - if args.verbose >= wikiget.VERY_VERBOSE: - print(f"User agent: {wikiget.USER_AGENT}") - +def query_api(filename: str, site_name: str, args: Namespace) -> Image: # connect to site and identify ourselves - if args.verbose >= wikiget.STD_VERBOSE: - print(f"Site name: {site_name}") + logging.info(f"Connecting to {site_name}") try: site = Site(site_name, path=args.path, clients_useragent=wikiget.USER_AGENT) if args.username and args.password: site.login(args.username, args.password) except ConnectionError as e: - # usually this means there is no such site, or there's no network - # connection, though it could be a certificate problem - print("Error: couldn't connect to specified site.") - if args.verbose >= wikiget.VERY_VERBOSE: - print("Full error message:") - print(e) - sys.exit(1) + # usually this means there is no such site, or there's no network connection, + # though it could be a certificate problem + logging.error("Could not connect to specified site") + logging.debug(e) + raise except HTTPError as e: # most likely a 403 forbidden or 404 not found error for api.php - print( - "Error: couldn't find the specified wiki's api.php. " - "Check the value of --path." + logging.error( + "Could not find the specified wiki's api.php. Check the value of --path." ) - if args.verbose >= wikiget.VERY_VERBOSE: - print("Full error message:") - print(e) - sys.exit(1) + logging.debug(e) + raise except (InvalidResponse, LoginError) as e: - # InvalidResponse: site exists, but we couldn't communicate with the - # API endpoint for some reason other than an HTTP error. + # InvalidResponse: site exists, but we couldn't communicate with the API + # endpoint for some reason other than an HTTP error. # LoginError: missing or invalid credentials - print(e) - sys.exit(1) + logging.error(e) + raise # get info about the target file try: - file = site.images[filename] + image = site.images[filename] except APIError as e: - # an API error at this point likely means access is denied, - # which could happen with a private wiki - print( - "Error: access denied. Try providing credentials with " - "--username and --password." + # an API error at this point likely means access is denied, which could happen + # with a private wiki + logging.error( + "Access denied. Try providing credentials with --username and --password." ) - if args.verbose >= wikiget.VERY_VERBOSE: - print("Full error message:") - for i in e.args: - print(i) - sys.exit(1) + for i in e.args: + logging.debug(i) + raise - if file.imageinfo != {}: - # file exists either locally or at a common repository, - # like Wikimedia Commons + return image + + +def prep_download(dl: str, args: Namespace) -> File: + file = get_dest(dl, args) + file.image = query_api(file.name, file.site, args) + return file + + +def batch_download(args: Namespace) -> int: + input_file = args.FILE + dl_list = {} + errors = 0 + + logging.info(f"Using batch file '{input_file}'.") + + try: + fd = open(input_file) + except OSError as e: + logging.error("File could not be read. The following error was encountered:") + logging.error(e) + sys.exit(1) + else: + with fd: + # read the file into memory and process each line as we go + for line_num, line in enumerate(fd, start=1): + line_s = line.strip() + # ignore blank lines and lines starting with "#" (for comments) + if line_s and not line_s.startswith("#"): + dl_list[line_num] = line_s + + # TODO: validate file contents before download process starts + with ThreadPoolExecutor(max_workers=args.threads) as executor: + futures = [] + for line_num, line in dl_list.items(): + # keep track of batch file line numbers for debugging/logging purposes + logging.info(f"Processing '{line}' at line {line_num}") + try: + file = prep_download(line, args) + except ParseError as e: + logging.warning(f"{e} (line {line_num})") + errors += 1 + continue + except (ConnectionError, HTTPError, InvalidResponse, LoginError, APIError): + logging.warning( + f"Unable to download '{line}' (line {line_num}) due to an error" + ) + errors += 1 + continue + future = executor.submit(download, file, args) + futures.append(future) + # wait for downloads to finish + for future in futures: + errors += future.result() + return errors + + +def download(f: File, args: Namespace) -> int: + file = f.image + filename = f.name + dest = f.dest + site = file.site + + errors = 0 + + logger = logging.getLogger("") + adapter = FileLogAdapter(logger, {"filename": filename}) + + if file.exists: + # file exists either locally or at a common repository, like Wikimedia Commons file_url = file.imageinfo["url"] file_size = file.imageinfo["size"] file_sha1 = file.imageinfo["sha1"] - if args.verbose >= wikiget.STD_VERBOSE: - print( - f"Info: downloading '{filename}' " - f"({file_size} bytes) from {site.host}", - end="", - ) - if args.output: - print(f" to '{dest}'") - else: - print("\n", end="") - print(f"Info: {file_url}") + filename_log = f"Downloading '{filename}' ({file_size} bytes) from {site.host}" + if args.output: + filename_log += f" to '{dest}'" + adapter.info(filename_log) + adapter.info(f"{file_url}") if os.path.isfile(dest) and not args.force: - print(f"File '{dest}' already exists, skipping download (use -f to ignore)") + adapter.warning("File already exists, skipping download (use -f to force)") + errors += 1 else: try: fd = open(dest, "wb") except OSError as e: - print("File could not be written. The following error was encountered:") - print(e) - sys.exit(1) + adapter.error(f"File could not be written. {e}") + errors += 1 else: # download the file(s) if args.verbose >= wikiget.STD_VERBOSE: @@ -154,21 +183,25 @@ def download(dl, args): fd.write(chunk) progress_bar.update(len(chunk)) - # verify file integrity and optionally print details + # verify file integrity and log details dl_sha1 = verify_hash(dest) - if args.verbose >= wikiget.STD_VERBOSE: - print(f"Info: downloaded file SHA1 is {dl_sha1}") - print(f"Info: server file SHA1 is {file_sha1}") + adapter.info(f"Remote file SHA1 is {file_sha1}") + adapter.info(f"Local file SHA1 is {dl_sha1}") if dl_sha1 == file_sha1: - if args.verbose >= wikiget.STD_VERBOSE: - print("Info: hashes match!") + adapter.info("Hashes match!") # at this point, we've successfully downloaded the file + success_log = f"'{filename}' downloaded" + if args.output: + success_log += f" to '{dest}'" + adapter.info(success_log) else: - print("Error: hash mismatch! Downloaded file may be corrupt.") - sys.exit(1) + adapter.error("Hash mismatch! Downloaded file may be corrupt.") + errors += 1 else: # no file information returned - print(f"Target '{filename}' does not appear to be a valid file.") - sys.exit(1) + adapter.warning("Target does not appear to be a valid file") + errors += 1 + + return errors diff --git a/src/wikiget/exceptions.py b/src/wikiget/exceptions.py new file mode 100644 index 0000000..94ed6b2 --- /dev/null +++ b/src/wikiget/exceptions.py @@ -0,0 +1,20 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2023 Cody Logan +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see <https://www.gnu.org/licenses/>. + + +class ParseError(Exception): + pass diff --git a/src/wikiget/file.py b/src/wikiget/file.py new file mode 100644 index 0000000..b890e63 --- /dev/null +++ b/src/wikiget/file.py @@ -0,0 +1,39 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2023 Cody Logan +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see <https://www.gnu.org/licenses/>. + +from mwclient.image import Image + +from wikiget import DEFAULT_SITE + + +class File: + def __init__(self, name: str, dest: str = "", site: str = "") -> None: + """ + Initializes a new file with the specified name and an optional destination name. + + :param name: name of the file + :type name: str + :param dest: destination of the file, if different from the name; if not + specified, defaults to the name + :type dest: str, optional + :param site: name of the site hosting the file; if not specified, defaults to + the global default site + """ + self.image: Image = None + self.name = name + self.dest = dest if dest else name + self.site = site if site else DEFAULT_SITE diff --git a/src/wikiget/logging.py b/src/wikiget/logging.py new file mode 100644 index 0000000..87b917c --- /dev/null +++ b/src/wikiget/logging.py @@ -0,0 +1,58 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2023 Cody Logan +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see <https://www.gnu.org/licenses/>. + +import logging +from argparse import Namespace + +import wikiget + + +class FileLogAdapter(logging.LoggerAdapter): + def process(self, msg, kwargs): + return f"[{self.extra['filename']}] {msg}", kwargs + + +def configure_logging(args: Namespace) -> None: + loglevel = logging.WARNING + if args.verbose >= wikiget.VERY_VERBOSE: + # this includes API and library messages + loglevel = logging.DEBUG + elif args.verbose >= wikiget.STD_VERBOSE: + loglevel = logging.INFO + elif args.quiet: + loglevel = logging.ERROR + + # configure logging: + # console log level is set via -v, -vv, and -q options; + # file log level is always debug (TODO: make this user configurable) + base_format = "%(message)s" + log_format = "[%(levelname)s] " + base_format + if args.logfile: + # log to console and file + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s [%(levelname)-7s] " + base_format, + filename=args.logfile, + ) + + console = logging.StreamHandler() + console.setLevel(loglevel) + console.setFormatter(logging.Formatter(log_format)) + logging.getLogger("").addHandler(console) + else: + # log only to console + logging.basicConfig(level=loglevel, format=log_format) diff --git a/src/wikiget/parse.py b/src/wikiget/parse.py new file mode 100644 index 0000000..fe3fe43 --- /dev/null +++ b/src/wikiget/parse.py @@ -0,0 +1,59 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2023 Cody Logan +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see <https://www.gnu.org/licenses/>. + +import logging +from argparse import Namespace +from urllib.parse import unquote, urlparse + +import wikiget +from wikiget.exceptions import ParseError +from wikiget.file import File +from wikiget.validations import valid_file + + +def get_dest(dl: str, args: Namespace) -> File: + url = urlparse(dl) + + if url.netloc: + filename = url.path + site_name = url.netloc + if args.site is not wikiget.DEFAULT_SITE: + # this will work even if the user specifies 'commons.wikimedia.org' since + # we're comparing objects instead of values (is not vs. !=) + logging.warning("Target is a URL, ignoring site specified with --site") + else: + filename = dl + site_name = args.site + + file_match = valid_file(filename) + + # check if this is a valid file + if file_match and file_match.group(1): + # has File:/Image: prefix and extension + filename = file_match.group(2) + else: + # no file extension and/or prefix, probably an article + msg = f"Could not parse input '{filename}' as a file" + raise ParseError(msg) + + filename = unquote(filename) # remove URL encoding for special characters + + dest = args.output or filename + + file = File(filename, dest, site_name) + + return file diff --git a/src/wikiget/validations.py b/src/wikiget/validations.py index dc70df4..c9e7bcf 100644 --- a/src/wikiget/validations.py +++ b/src/wikiget/validations.py @@ -1,5 +1,5 @@ # wikiget - CLI tool for downloading files from Wikimedia sites -# Copyright (C) 2018, 2019, 2020 Cody Logan +# Copyright (C) 2018-2023 Cody Logan # SPDX-License-Identifier: GPL-3.0-or-later # # Wikiget is free software: you can redistribute it and/or modify @@ -17,17 +17,21 @@ import hashlib import re +from typing import Optional from wikiget import BLOCKSIZE -def valid_file(search_string): +def valid_file(search_string: str) -> Optional[re.Match]: """ - Determines if the given string contains a valid file name, defined as a - string ending with a '.' and at least one character, beginning with 'File:' - or 'Image:', the standard file prefixes in MediaWiki. + Determines if the given string contains a valid file name, defined as a string + ending with a '.' and at least one character, beginning with 'File:' or 'Image:', + the standard file prefixes in MediaWiki. + :param search_string: string to validate + :type search_string: str :returns: a regex Match object if there's a match or None otherwise + :rtype: re.Match """ # second group could also restrict to file extensions with three or more # letters with ([^/\r\n\t\f\v]+\.\w{3,}) @@ -35,25 +39,30 @@ def valid_file(search_string): return file_regex.search(search_string) -def valid_site(search_string): +def valid_site(search_string: str) -> Optional[re.Match]: """ - Determines if the given string contains a valid site name, defined as a - string ending with 'wikipedia.org' or 'wikimedia.org'. This covers all - subdomains of those domains. Eventually, it should be possible to support - any MediaWiki site, regardless of domain name. + Determines if the given string contains a valid site name, defined as a string + ending with 'wikipedia.org' or 'wikimedia.org'. This covers all subdomains of those + domains. Eventually, it should be possible to support any MediaWiki site, regardless + of domain name. + :param search_string: string to validate + :type search_string: str :returns: a regex Match object if there's a match or None otherwise + :rtype: re.Match """ site_regex = re.compile(r"wiki[mp]edia\.org$", re.I) return site_regex.search(search_string) -def verify_hash(filename): +def verify_hash(filename: str) -> str: """ - Calculates the SHA1 hash of the given file for comparison with a known - value. + Calculates the SHA1 hash of the given file for comparison with a known value. + :param filename: name of the file to calculate a hash for + :type filename: str :return: hash digest + :rtype: str """ hasher = hashlib.sha1() # noqa: S324 with open(filename, "rb") as dl: diff --git a/src/wikiget/version.py b/src/wikiget/version.py index dd9b22c..34dabb7 100644 --- a/src/wikiget/version.py +++ b/src/wikiget/version.py @@ -1 +1,18 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2018-2023 Cody Logan and contributors +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see <https://www.gnu.org/licenses/>. + __version__ = "0.5.1" diff --git a/src/wikiget/wikiget.py b/src/wikiget/wikiget.py index ba36766..e64d00e 100644 --- a/src/wikiget/wikiget.py +++ b/src/wikiget/wikiget.py @@ -1,5 +1,5 @@ # wikiget - CLI tool for downloading files from Wikimedia sites -# Copyright (C) 2018-2021 Cody Logan and contributors +# Copyright (C) 2018-2023 Cody Logan and contributors # SPDX-License-Identifier: GPL-3.0-or-later # # Wikiget is free software: you can redistribute it and/or modify @@ -19,38 +19,33 @@ import argparse import logging import sys -import wikiget -from wikiget.dl import download +from mwclient import APIError, InvalidResponse, LoginError +from requests import ConnectionError, HTTPError +import wikiget +from wikiget.dl import batch_download, download, prep_download +from wikiget.exceptions import ParseError +from wikiget.logging import configure_logging -def main(): - """ - Main entry point for console script. Automatically compiled by setuptools - when installed with `pip install` or `python setup.py install`. - """ +def construct_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=""" - A tool for downloading files from - MediaWiki sites using the file name or + A tool for downloading files from MediaWiki sites using the file name or description page URL """, epilog=""" - Copyright (C) 2018-2023 Cody Logan - and contributors. - License GPLv3+: GNU GPL version 3 or later - <http://www.gnu.org/licenses/gpl.html>. - This is free software; you are free to - change and redistribute it under certain - conditions. There is NO WARRANTY, to the - extent permitted by law. + Copyright (C) 2018-2023 Cody Logan and contributors. License GPLv3+: GNU GPL + version 3 or later <http://www.gnu.org/licenses/gpl.html>. This is free + software; you are free to change and redistribute it under certain conditions. + There is NO WARRANTY, to the extent permitted by law. """, ) parser.add_argument( "FILE", help=""" - name of the file to download with the File: - prefix, or the URL of its file description page + name of the file to download with the File: prefix, or the URL of its file + description page """, ) parser.add_argument( @@ -80,52 +75,76 @@ def main(): help="MediaWiki site to download from (default: %(default)s)", ) parser.add_argument( - "-p", + "-P", "--path", default=wikiget.DEFAULT_PATH, help="MediaWiki site path, where api.php is located (default: %(default)s)", ) parser.add_argument( - "--username", default="", help="MediaWiki site username, for private wikis" + "-u", + "--username", + default="", + help="MediaWiki site username, for private wikis", ) parser.add_argument( - "--password", default="", help="MediaWiki site password, for private wikis" + "-p", + "--password", + default="", + help="MediaWiki site password, for private wikis", ) output_options = parser.add_mutually_exclusive_group() output_options.add_argument("-o", "--output", help="write download to OUTPUT") output_options.add_argument( "-a", "--batch", - help="treat FILE as a textfile containing " - "multiple files to download, one URL or " - "filename per line", + help="treat FILE as a textfile containing multiple files to download, one URL " + "or filename per line", action="store_true", ) + parser.add_argument( + "-l", "--logfile", default="", help="save log output to LOGFILE" + ) + parser.add_argument( + "-j", + "--threads", + default=1, + help="number of parallel downloads to attempt in batch mode", + type=int, + ) + return parser + + +def main() -> None: + # setup our environment + parser = construct_parser() args = parser.parse_args() + configure_logging(args) - # print API and debug messages in verbose mode - if args.verbose >= wikiget.VERY_VERBOSE: - logging.basicConfig(level=logging.DEBUG) - elif args.verbose >= wikiget.STD_VERBOSE: - logging.basicConfig(level=logging.WARNING) + # log events are appended to the file if it already exists, so note the start of a + # new download session + logging.info(f"Starting download session using wikiget {wikiget.wikiget_version}") + logging.debug(f"User agent: {wikiget.USER_AGENT}") if args.batch: # batch download mode - input_file = args.FILE - if args.verbose >= wikiget.STD_VERBOSE: - print(f"Info: using batch file '{input_file}'") - try: - fd = open(input_file) - except OSError as e: - print("File could not be read. The following error was encountered:") - print(e) + errors = batch_download(args) + if errors: + # return non-zero exit code if any problems were encountered, even if some + # downloads completed successfully + logging.warning( + f"{errors} problem{'s'[:errors^1]} encountered during batch processing" + ) sys.exit(1) - else: - with fd: - for _, line in enumerate(fd): - download(line.strip(), args) else: # single download mode - dl = args.FILE - download(dl, args) + try: + file = prep_download(args.FILE, args) + except ParseError as e: + logging.error(e) + sys.exit(1) + except (ConnectionError, HTTPError, InvalidResponse, LoginError, APIError): + sys.exit(1) + errors = download(file, args) + if errors: + sys.exit(1) diff --git a/tests/test_dl.py b/tests/test_dl.py new file mode 100644 index 0000000..fc68733 --- /dev/null +++ b/tests/test_dl.py @@ -0,0 +1,46 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2023 Cody Logan +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see <https://www.gnu.org/licenses/>. + +import logging + +import pytest + +from wikiget import USER_AGENT +from wikiget.dl import prep_download, query_api +from wikiget.wikiget import construct_parser + + +# TODO: don't hit the actual API when doing tests +@pytest.mark.skip +class TestQueryApi: + parser = construct_parser() + + def test_query_api(self, caplog): + caplog.set_level(logging.DEBUG) + args = self.parser.parse_args(["File:Example.jpg"]) + file, site = query_api("Example.jpg", "commons.wikimedia.org", args) + assert USER_AGENT in caplog.text + + +@pytest.mark.skip +class TestPrepDownload: + parser = construct_parser() + + def test_prep_download(self): + args = self.parser.parse_args(["File:Example.jpg"]) + file = prep_download(args.FILE, args) + assert file is not None diff --git a/tests/test_file_class.py b/tests/test_file_class.py new file mode 100644 index 0000000..dd30207 --- /dev/null +++ b/tests/test_file_class.py @@ -0,0 +1,37 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2023 Cody Logan +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see <https://www.gnu.org/licenses/>. + +from wikiget import DEFAULT_SITE +from wikiget.file import File + + +def test_file_with_name_only(): + file = File("foobar.jpg") + assert file.name == "foobar.jpg" + assert file.dest == file.name + assert file.site == DEFAULT_SITE + + +def test_file_with_name_and_dest(): + file = File("foobar.jpg", dest="bazqux.jpg") + assert file.dest == "bazqux.jpg" + assert file.dest != file.name + + +def test_file_with_name_and_site(): + file = File("foobar.jpg", site="en.wikipedia.org") + assert file.site == "en.wikipedia.org" diff --git a/tests/test_parse.py b/tests/test_parse.py new file mode 100644 index 0000000..757b361 --- /dev/null +++ b/tests/test_parse.py @@ -0,0 +1,60 @@ +# wikiget - CLI tool for downloading files from Wikimedia sites +# Copyright (C) 2023 Cody Logan +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Wikiget is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Wikiget is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Wikiget. If not, see <https://www.gnu.org/licenses/>. + +import pytest + +from wikiget.exceptions import ParseError +from wikiget.parse import get_dest +from wikiget.wikiget import construct_parser + + +class TestGetDest: + parser = construct_parser() + + def test_get_dest_with_filename(self): + args = self.parser.parse_args(["File:Example.jpg"]) + file = get_dest(args.FILE, args) + assert file.name == "Example.jpg" + assert file.dest == "Example.jpg" + assert file.site == "commons.wikimedia.org" + + def test_get_dest_with_url(self): + args = self.parser.parse_args( + [ + "https://en.wikipedia.org/wiki/File:Example.jpg", + ] + ) + file = get_dest(args.FILE, args) + assert file.name == "Example.jpg" + assert file.dest == "Example.jpg" + assert file.site == "en.wikipedia.org" + + def test_get_dest_with_bad_filename(self): + args = self.parser.parse_args(["Example.jpg"]) + with pytest.raises(ParseError): + _ = get_dest(args.FILE, args) + + def test_get_dest_with_different_site(self, caplog: pytest.LogCaptureFixture): + args = self.parser.parse_args( + [ + "https://commons.wikimedia.org/wiki/File:Example.jpg", + "--site", + "commons.wikimedia.org", + ] + ) + _ = get_dest(args.FILE, args) + assert "Target is a URL, ignoring site specified with --site" in caplog.text diff --git a/tests/test_validations.py b/tests/test_validations.py index 1abd96a..9d70f6e 100644 --- a/tests/test_validations.py +++ b/tests/test_validations.py @@ -1,5 +1,5 @@ # wikiget - CLI tool for downloading files from Wikimedia sites -# Copyright (C) 2018-2021 Cody Logan +# Copyright (C) 2018-2023 Cody Logan # SPDX-License-Identifier: GPL-3.0-or-later # # Wikiget is free software: you can redistribute it and/or modify |