From 0dd3f2e1378901db5ce9f851eeb7cd2671acd962 Mon Sep 17 00:00:00 2001 From: Lab-OsintItalia Date: Sun, 3 Oct 2021 10:42:04 +0200 Subject: [PATCH 01/72] added snapcraft and bin folder --- bin/maigret | 18 ++++++++++++++++++ snapcraft.yaml | 30 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100755 bin/maigret create mode 100644 snapcraft.yaml diff --git a/bin/maigret b/bin/maigret new file mode 100755 index 000000000..76d98a8d7 --- /dev/null +++ b/bin/maigret @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import asyncio +import sys + +from maigret.maigret import main + + +def run(): + try: + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + except KeyboardInterrupt: + print('Maigret is interrupted.') + sys.exit(1) + + +if __name__ == "__main__": + run() diff --git a/snapcraft.yaml b/snapcraft.yaml new file mode 100644 index 000000000..b1daa35f3 --- /dev/null +++ b/snapcraft.yaml @@ -0,0 +1,30 @@ +name: maigret +version: git +summary: v +description: | + Test Test Test +base: core18 +confinement: strict + + +parts: + maigret: + plugin: python + python-version: python3 + source: . + stage-packages: + - python-six + + +apps: + maigret: + command: bin/maigret + + +architectures: + - build-on: amd64 + - build-on: i386 + + + + From 998ff2e4e6c3f46bc874e6f39cc44905aeca1fa5 Mon Sep 17 00:00:00 2001 From: Lab-Osintitalia <88847401+Lab-Osintitalia@users.noreply.github.com> Date: Sun, 3 Oct 2021 10:44:13 +0200 Subject: [PATCH 02/72] Update snapcraft.yaml --- snapcraft.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index b1daa35f3..481989c5b 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -1,4 +1,4 @@ -name: maigret +name: maigret2 version: git summary: v description: | @@ -8,7 +8,7 @@ confinement: strict parts: - maigret: + maigret2: plugin: python python-version: python3 source: . @@ -17,7 +17,7 @@ parts: apps: - maigret: + maigret2: command: bin/maigret From c147f19c3a85f2a81751e6cb501a3db4784584f0 Mon Sep 17 00:00:00 2001 From: Lab-Osintitalia <88847401+Lab-Osintitalia@users.noreply.github.com> Date: Sun, 3 Oct 2021 10:44:38 +0200 Subject: [PATCH 03/72] Update snapcraft.yaml --- snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index 481989c5b..b87e27791 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -1,6 +1,6 @@ name: maigret2 version: git -summary: v +summary: [ SOCMINT | Instagram ] description: | Test Test Test base: core18 From c57204ff2f222e316bba7ea01b3b2b2a4a1c5a66 Mon Sep 17 00:00:00 2001 From: Lab-Osintitalia <88847401+Lab-Osintitalia@users.noreply.github.com> Date: Sun, 3 Oct 2021 10:53:16 +0200 Subject: [PATCH 04/72] Update snapcraft.yaml --- snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index b87e27791..38243561b 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -1,6 +1,6 @@ name: maigret2 version: git -summary: [ SOCMINT | Instagram ] +summary: SOCMINT / Instagram description: | Test Test Test base: core18 From e3b4512c47345cf64ceb911522b4fd486dc67b68 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 31 Oct 2021 19:37:26 +0300 Subject: [PATCH 05/72] Delayed import of requests module, speed check command, reqs updated --- Makefile | 6 ++++++ maigret/activation.py | 7 ++++++- maigret/checking.py | 4 +--- maigret/sites.py | 4 ++-- requirements.txt | 7 ++----- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 33f4da364..903454349 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,12 @@ lint: @echo 'mypy' mypy ${LINT_FILES} +speed: + time python3 ./maigret.py --version + python3 -c "import timeit; t = timeit.Timer('import maigret'); print(t.timeit(number = 1000000))" + python3 -X importtime -c "import maigret" 2> maigret-import.log + python3 -m tuna maigret-import.log + format: @echo 'black' black --skip-string-normalization ${LINT_FILES} diff --git a/maigret/activation.py b/maigret/activation.py index 77fe394b6..9ad9d4c10 100644 --- a/maigret/activation.py +++ b/maigret/activation.py @@ -1,7 +1,6 @@ from http.cookiejar import MozillaCookieJar from http.cookies import Morsel -import requests from aiohttp import CookieJar @@ -10,6 +9,8 @@ class ParsingActivator: def twitter(site, logger, cookies={}): headers = dict(site.headers) del headers["x-guest-token"] + import requests + r = requests.post(site.activation["url"], headers=headers) logger.info(r) j = r.json() @@ -21,6 +22,8 @@ def vimeo(site, logger, cookies={}): headers = dict(site.headers) if "Authorization" in headers: del headers["Authorization"] + import requests + r = requests.get(site.activation["url"], headers=headers) jwt_token = r.json()["jwt"] site.headers["Authorization"] = "jwt " + jwt_token @@ -30,6 +33,8 @@ def spotify(site, logger, cookies={}): headers = dict(site.headers) if "Authorization" in headers: del headers["Authorization"] + import requests + r = requests.get(site.activation["url"]) bearer_token = r.json()["accessToken"] site.headers["authorization"] = f"Bearer {bearer_token}" diff --git a/maigret/checking.py b/maigret/checking.py index 163316e27..08c42d720 100644 --- a/maigret/checking.py +++ b/maigret/checking.py @@ -63,9 +63,7 @@ def __init__(self, *args, **kwargs): from aiohttp_socks import ProxyConnector # make http client session - connector = ( - ProxyConnector.from_url(proxy) if proxy else TCPConnector(ssl=False) - ) + connector = ProxyConnector.from_url(proxy) if proxy else TCPConnector(ssl=False) connector.verify_ssl = False self.session = ClientSession( connector=connector, trust_env=True, cookie_jar=cookie_jar diff --git a/maigret/sites.py b/maigret/sites.py index 32bc316e2..5d40076b1 100644 --- a/maigret/sites.py +++ b/maigret/sites.py @@ -5,8 +5,6 @@ import sys from typing import Optional, List, Dict, Any, Tuple -import requests - from .utils import CaseConverter, URLMatcher, is_country_tag @@ -359,6 +357,8 @@ def load_from_http(self, url: str) -> "MaigretDatabase": if not is_url_valid: raise FileNotFoundError(f"Invalid data file URL '{url}'.") + import requests + try: response = requests.get(url=url) except Exception as error: diff --git a/requirements.txt b/requirements.txt index 9fbf6e252..daad195a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,12 @@ aiodns==3.0.0 -aiohttp==3.7.4 +aiohttp==3.7.4.post0 aiohttp-socks==0.5.5 arabic-reshaper==2.1.1 async-timeout==3.0.1 attrs==20.3.0 -beautifulsoup4==4.9.3 -bs4==0.0.1 certifi==2020.12.5 chardet==3.0.4 colorama==0.4.4 -python-dateutil==2.8.1 future==0.18.2 future-annotations==1.0.0 html5lib==1.1 @@ -24,7 +21,7 @@ PyPDF2==1.26.0 PySocks==1.7.1 python-bidi==0.4.2 python-socks==1.1.2 -requests>=2.24.0 +requests==2.26.0 requests-futures==1.0.0 six==1.15.0 socid-extractor>=0.0.21 From 8f86d76db686d8fd9d69d11fb866e6c37c86dc6c Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Sun, 31 Oct 2021 20:35:16 +0300 Subject: [PATCH 06/72] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..3c73d6bda --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,70 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main ] + schedule: + - cron: '23 6 * * 1' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 516861e0aedca3d2b0053a84e83d44db85bfe66d Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 31 Oct 2021 21:38:32 +0300 Subject: [PATCH 07/72] Some security fixes Added link to Patreon --- .github/FUNDING.yml | 3 +++ requirements.txt | 2 +- tests/test_utils.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..fcda0d32b --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +patreon: soxoj diff --git a/requirements.txt b/requirements.txt index daad195a4..60bcc92ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ aiodns==3.0.0 -aiohttp==3.7.4.post0 +aiohttp==3.7.4 aiohttp-socks==0.5.5 arabic-reshaper==2.1.1 async-timeout==3.0.1 diff --git a/tests/test_utils.py b/tests/test_utils.py index fb6252e65..0ff780193 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -73,7 +73,7 @@ def test_url_extract_main_part(): ['/', ''], ] - url_regexp = re.compile('^https?://(www.)?flickr.com/photos/(.+?)$') + url_regexp = re.compile(r'^https?://(www\.)?flickr.com/photos/(.+?)$') # combine parts variations for url_parts in itertools.product(*parts): url = ''.join(url_parts) From eeab6ba82cc07ac48f8a6b2eeeae9a70a26995dc Mon Sep 17 00:00:00 2001 From: Egor Nagornov Date: Tue, 2 Nov 2021 17:21:27 +0700 Subject: [PATCH 08/72] Move wiki pages to ReadTheDocs --- docs/Makefile | 20 +++ docs/make.bat | 35 +++++ docs/source/command-line-options.rst | 124 ++++++++++++++++++ docs/source/conf.py | 36 +++++ .../extracting-information-from-pages.rst | 35 +++++ docs/source/features.rst | 76 +++++++++++ docs/source/index.rst | 29 ++++ docs/source/philosophy.rst | 6 + docs/source/roadmap.rst | 18 +++ docs/source/supported-identifier-types.rst | 15 +++ docs/source/tags.rst | 24 ++++ docs/source/usage-examples.rst | 53 ++++++++ 12 files changed, 471 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/source/command-line-options.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/extracting-information-from-pages.rst create mode 100644 docs/source/features.rst create mode 100644 docs/source/index.rst create mode 100644 docs/source/philosophy.rst create mode 100644 docs/source/roadmap.rst create mode 100644 docs/source/supported-identifier-types.rst create mode 100644 docs/source/tags.rst create mode 100644 docs/source/usage-examples.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..d0c3cbf10 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 000000000..9534b0181 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/command-line-options.rst b/docs/source/command-line-options.rst new file mode 100644 index 000000000..44dcd8f62 --- /dev/null +++ b/docs/source/command-line-options.rst @@ -0,0 +1,124 @@ +.. _command-line-options: + +Command line options +==================== + +Usernames +--------- + +``maigret username1 username2 ...`` + +You can specify several usernames separated by space. Usernames are +**not** mandatory as there are other operations modes (see below). + +Parsing of account pages and online documents +--------------------------------------------- + +``maigret --parse URL`` + +Maigret will try to extract information about the document/account owner +(including username and other ids) and will make a search by the +extracted username and ids. :doc:`Examples `. + +Main options +------------ + +``--tags`` - Filter sites for searching by tags: sites categories and +two-letter country codes. E.g. photo, dating, sport; jp, us, global. +Multiple tags can be associated with one site. **Warning: tags markup is +not stable now.** + +``-n``, ``--max-connections`` - Allowed number of concurrent connections +**(default: 100)**. + +``-a``, ``--all-sites`` - Use all sites for scan **(default: top 500)**. + +``--top-sites`` - Count of sites for scan ranked by Alexa Top +**(default: top 500)**. + +``--timeout`` - Time (in seconds) to wait for responses from sites +**(default: 30)**. A longer timeout will be more likely to get results +from slow sites. On the other hand, this may cause a long delay to +gather all results. The choice of the right timeout should be carried +out taking into account the bandwidth of the Internet connection. + +``--cookies-jar-file`` - File with custom cookies in Netscape format +(aka cookies.txt). You can install an extension to your browser to +download own cookies (`Chrome `_, `Firefox `_). + +``--no-recursion`` - Disable parsing pages for other usernames and +recursive search by them. + +``--use-disabled-sites`` - Use disabled sites to search (may cause many +false positives). + +``--id-type`` - Specify identifier(s) type (default: username). +Supported types: gaia_id, vk_id, yandex_public_id, ok_id, wikimapia_uid. +Currently, you must add ``-a`` flag to run a scan on sites with custom +id types, sites will be filtered automatically. + +``--ignore-ids`` - Do not make search by the specified username or other +ids. Useful for repeated scanning with found known irrelevant usernames. + +``--db`` - Load Maigret database from a JSON file or an online, valid, +JSON file. + +``--retries RETRIES`` - Count of attempts to restart temporarily failed +requests. + +Reports +------- + +``-P``, ``--pdf`` - Generate a PDF report (general report on all +usernames). + +``-H``, ``--html`` - Generate an HTML report file (general report on all +usernames). + +``-X``, ``--xmind`` - Generate an XMind 8 mindmap (one report per +username). + +``-C``, ``--csv`` - Generate a CSV report (one report per username). + +``-T``, ``--txt`` - Generate a TXT report (one report per username). + +``-J``, ``--json`` - Generate a JSON report of specific type: simple, +ndjson (one report per username). E.g. ``--json ndjson`` + +``-fo``, ``--folderoutput`` - Results will be saved to this folder, +``results`` by default. Will be created if doesn’t exist. + +Output options +-------------- + +``-v``, ``--verbose`` - Display extra information and metrics. +*(loglevel=WARNING)* + +``-vv``, ``--info`` - Display service information. *(loglevel=INFO)* + +``-vvv``, ``--debug``, ``-d`` - Display debugging information and site +responses. *(loglevel=DEBUG)* + +``--print-not-found`` - Print sites where the username was not found. + +``--print-errors`` - Print errors messages: connection, captcha, site +country ban, etc. + +Other operations modes +---------------------- + +``--version`` - Display version information and dependencies. + +``--self-check`` - Do self-checking for sites and database and disable +non-working ones **for current search session** by default. It’s useful +for testing new internet connection (it depends on provider/hosting on +which sites there will be censorship stub or captcha display). After +checking Maigret asks if you want to save updates, answering y/Y will +rewrite the local database. + +``--submit URL`` - Do an automatic analysis of the given account URL or +site main page URL to determine the site engine and methods to check +account presence. After checking Maigret asks if you want to add the +site, answering y/Y will rewrite the local database. + + diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 000000000..a0d4a62d6 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,36 @@ +# Configuration file for the Sphinx documentation builder. + +# -- Project information + +project = 'Maigret' +copyright = '2021, soxoj' +author = 'soxoj' + +release = '0.3.1' +version = '0.3.1' + +# -- General configuration + +extensions = [ + 'sphinx.ext.duration', + 'sphinx.ext.doctest', + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.intersphinx', + 'sphinx_copybutton' +] + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3/', None), + 'sphinx': ('https://www.sphinx-doc.org/en/master/', None), +} +intersphinx_disabled_domains = ['std'] + +templates_path = ['_templates'] + +# -- Options for HTML output + +html_theme = 'sphinx_rtd_theme' + +# -- Options for EPUB output +epub_show_urls = 'footnote' diff --git a/docs/source/extracting-information-from-pages.rst b/docs/source/extracting-information-from-pages.rst new file mode 100644 index 000000000..c24e6d59c --- /dev/null +++ b/docs/source/extracting-information-from-pages.rst @@ -0,0 +1,35 @@ +.. _extracting-information-from-pages: + +Extracting information from pages +================================= +Maigret can parse URLs and content of web pages by URLs to extract info about account owner and other meta information. + +You must specify the URL with the option ``--parse``, it's can be a link to an account or an online document. List of supported sites `see here `_. + +After the end of the parsing phase, Maigret will start the search phase by :doc:`supported identifiers ` found (usernames, ids, etc.). + +Examples +-------- +.. code-block:: console + + $ maigret --parse https://docs.google.com/spreadsheets/d/1HtZKMLRXNsZ0HjtBmo0Gi03nUPiJIA4CC4jTYbCAnXw/edit\#gid\=0 + + Scanning webpage by URL https://docs.google.com/spreadsheets/d/1HtZKMLRXNsZ0HjtBmo0Gi03nUPiJIA4CC4jTYbCAnXw/edit#gid=0... + ┣╸org_name: Gooten + ┗╸mime_type: application/vnd.google-apps.ritz + Scanning webpage by URL https://clients6.google.com/drive/v2beta/files/1HtZKMLRXNsZ0HjtBmo0Gi03nUPiJIA4CC4jTYbCAnXw?fields=alternateLink%2CcopyRequiresWriterPermission%2CcreatedDate%2Cdescription%2CdriveId%2CfileSize%2CiconLink%2Cid%2Clabels(starred%2C%20trashed)%2ClastViewedByMeDate%2CmodifiedDate%2Cshared%2CteamDriveId%2CuserPermission(id%2Cname%2CemailAddress%2Cdomain%2Crole%2CadditionalRoles%2CphotoLink%2Ctype%2CwithLink)%2Cpermissions(id%2Cname%2CemailAddress%2Cdomain%2Crole%2CadditionalRoles%2CphotoLink%2Ctype%2CwithLink)%2Cparents(id)%2Ccapabilities(canMoveItemWithinDrive%2CcanMoveItemOutOfDrive%2CcanMoveItemOutOfTeamDrive%2CcanAddChildren%2CcanEdit%2CcanDownload%2CcanComment%2CcanMoveChildrenWithinDrive%2CcanRename%2CcanRemoveChildren%2CcanMoveItemIntoTeamDrive)%2Ckind&supportsTeamDrives=true&enforceSingleParent=true&key=AIzaSyC1eQ1xj69IdTMeii5r7brs3R90eck-m7k... + ┣╸created_at: 2016-02-16T18:51:52.021Z + ┣╸updated_at: 2019-10-23T17:15:47.157Z + ┣╸gaia_id: 15696155517366416778 + ┣╸fullname: Nadia Burgess + ┣╸email: nadia@gooten.com + ┣╸image: https://lh3.googleusercontent.com/a-/AOh14GheZe1CyNa3NeJInWAl70qkip4oJ7qLsD8vDy6X=s64 + ┗╸email_username: nadia + +.. code-block:: console + + $ maigret.py --parse https://steamcommunity.com/profiles/76561199113454789 + Scanning webpage by URL https://steamcommunity.com/profiles/76561199113454789... + ┣╸steam_id: 76561199113454789 + ┣╸nickname: Pok + ┗╸username: Machine42 diff --git a/docs/source/features.rst b/docs/source/features.rst new file mode 100644 index 000000000..6367d5ded --- /dev/null +++ b/docs/source/features.rst @@ -0,0 +1,76 @@ +.. _features: + +Features +======== + +This is the list of Maigret features. + +Personal info gathering +----------------------- + +Maigret does the `parsing of accounts webpages and extraction `_ of personal info, links to other profiles, etc. +Extracted info displayed as an additional result in CLI output and as tables in HTML and PDF reports. +Also, Maigret use found ids and usernames from links to start a recursive search. + +Enabled by default, can be disabled with ``--no extracting``. + +Recursive search +---------------- + +Maigret can extract some :ref:`common ids ` and usernames from links on the account page (often people placed links to their other accounts) and immediately start new searches. All the gathered information will be displayed in CLI output and reports. + +Enabled by default, can be disabled with ``--no-recursion``. + +Reports +------- + +Maigret currently supports HTML, PDF, TXT, XMind mindmap, and JSON reports. + +HTML/PDF reports contain: + +- profile photo +- all the gathered personal info +- additional information about supposed personal data (full name, gender, location), resulting from statistics of all found accounts + +Also, there is a short text report in the CLI output after the end of a searching phase. + +Tags +---- + +The Maigret sites database very big (and will be bigger), and it is maybe an overhead to run a search for all the sites. +Also, it is often hard to understand, what sites more interesting for us in the case of a certain person. + +Tags markup allows selecting a subset of sites by interests (photo, messaging, finance, etc.) or by country. Tags of found accounts grouped and displayed in the reports. + +See full description :doc:`in the Tags Wiki page `. + +Censorship and captcha detection +-------------------------------- + +Maigret can detect common errors such as censorship stub pages, CloudFlare captcha pages, and others. +If you get more them 3% errors of a certain type in a session, you've got a warning message in the CLI output with recommendations to improve performance and avoid problems. + +Retries +------- + +Maigret will do retries of the requests with temporary errors got (connection failures, proxy errors, etc.). + +One attempt by default, can be changed with option ``--retries N``. + +Archives and mirrors checking +----------------------------- + +The Maigret database contains not only the original websites, but also mirrors, archives, and aggregators. For example: + +- `Reddit BigData search `_ +- `Picuki `_, Instagram mirror +- `Twitter shadowban `_ checker + +It allows getting additional info about the person and checking the existence of the account even if the main site is unavailable (bot protection, captcha, etc.) + +Simple API +---------- + +Maigret can be easily integrated with the use of Python package `maigret `_. + +Example: the official `Telegram bot `_ diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 000000000..ec6bda64d --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,29 @@ +.. _index: + +Welcome to the Maigret docs! +============================ + +**Maigret** is an easy-to-use and powerful OSINT tool for collecting a dossier on a person by username only. + +This is achieved by checking for accounts on a huge number of sites and gathering all the available information from web pages. + +The project's main goal - give to OSINT researchers and pentesters a **universal tool** to get maximum information about a subject and integrate it with other tools in automatization pipelines. + +You may be interested in: +------------------------- +- :doc:`Command line options description ` and :doc:`usage examples ` +- :doc:`Features list ` +- :doc:`Project roadmap ` + +.. toctree:: + :hidden: + :caption: Sections + + command-line-options + extracting-information-from-pages + features + philosophy + roadmap + supported-identifier-types + tags + usage-examples diff --git a/docs/source/philosophy.rst b/docs/source/philosophy.rst new file mode 100644 index 000000000..bd533124a --- /dev/null +++ b/docs/source/philosophy.rst @@ -0,0 +1,6 @@ +.. _philosophy: + +Philosophy +========== + +Username => Dossier diff --git a/docs/source/roadmap.rst b/docs/source/roadmap.rst new file mode 100644 index 000000000..c1dca395e --- /dev/null +++ b/docs/source/roadmap.rst @@ -0,0 +1,18 @@ +.. _roadmap: + +Roadmap +======= + +.. figure:: https://i.imgur.com/kk8cFdR.png + :target: https://i.imgur.com/kk8cFdR.png + :align: center + +Current status +-------------- + +- Sites DB stats - ok +- Scan sessions stats - ok +- Site engine autodetect - ok +- Engines for all the sites - WIP +- Unified reporting flow - ok +- Retries - ok diff --git a/docs/source/supported-identifier-types.rst b/docs/source/supported-identifier-types.rst new file mode 100644 index 000000000..312a22398 --- /dev/null +++ b/docs/source/supported-identifier-types.rst @@ -0,0 +1,15 @@ +.. _supported-identifier-types: + +Supported identifier types +========================== + +Maigret can search against not only ordinary usernames, but also through certain common identifiers. There is a list of all currently supported identifiers. + +- **gaia_id** - Google inner numeric user identifier, in former times was placed in a Google Plus account URL. +- **steam_id** - Steam inner numeric user identifier. +- **wikimapia_uid** - Wikimapia.org inner numeric user identifier. +- **uidme_uguid** - uID.me inner numeric user identifier. +- **yandex_public_id** - Yandex sites inner letter user identifier. See also: `YaSeeker `_. +- **vk_id** - VK.com inner numeric user identifier. +- **ok_id** - OK.ru inner numeric user identifier. +- **yelp_userid** - Yelp inner user identifier. diff --git a/docs/source/tags.rst b/docs/source/tags.rst new file mode 100644 index 000000000..b3d3f4b20 --- /dev/null +++ b/docs/source/tags.rst @@ -0,0 +1,24 @@ +.. _tags: + +Tags +==== + +The use of tags allows you to select a subset of the sites from big Maigret DB for search. + +**Warning: tags markup is not stable now.** + +There are several types of tags: + +1. **Country codes**: ``us``, ``jp``, ``br``... (`ISO 3166-1 alpha-2 `_). These tags reflect the site language and regional origin of its users and are then used to locate the owner of a username. If the regional origin is difficult to establish or a site is positioned as worldwide, `no country code is given`. There could be multiple country code tags for one site. + +2. **Site engines**. Most of them are forum engines now: ``uCoz``, ``vBulletin``, ``XenForo`` et al. Full list of engines stored in the Maigret database. + +3. **Sites' subject/type and interests of its users**. Full list of "standard" tags is `present in the source code `_ only for a moment. + +Usage +----- +``--tags en,jp`` -- search on US and Japanese sites (actually marked as such in the Maigret database) + +``--tags coding`` -- search on sites related to software development. + +``--tags ucoz`` -- search on uCoz sites only (mostly CIS countries) diff --git a/docs/source/usage-examples.rst b/docs/source/usage-examples.rst new file mode 100644 index 000000000..4aa6fa6a3 --- /dev/null +++ b/docs/source/usage-examples.rst @@ -0,0 +1,53 @@ +.. _usage-examples: + +Usage examples +============== + +Start a search for accounts with username ``machine42`` on top 500 sites from the Maigret DB. + +.. code-block:: console + + maigret machine42 + +Start a search for accounts with username ``machine42`` on **all sites** from the Maigret DB. + +.. code-block:: console + + maigret machine42 -a + +Start a search [...] and generate HTML and PDF reports. + +.. code-block:: console + + maigret machine42 -a -HP + +Start a search for accounts with username ``machine42`` only on Facebook. + +.. code-block:: console + + maigret machine42 --site Facebook + +Extract information from the Steam page by URL and start a search for accounts with found username ``machine42``. + +.. code-block:: console + + maigret --parse https://steamcommunity.com/profiles/76561199113454789 + +Start a search for accounts with username ``machine42`` only on US and Japanese sites. + +.. code-block:: console + + maigret michael --tags en,jp + +Start a search for accounts with username ``machine42`` only on sites related to software development. + +.. code-block:: console + + maigret michael --tags coding + +Start a search for accounts with username ``machine42`` on uCoz sites only (mostly CIS countries). + +.. code-block:: console + + maigret michael --tags ucoz + From 4ce241893b1b9f0c2329387bd6e55d6d87ef8827 Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Tue, 2 Nov 2021 20:22:00 +0300 Subject: [PATCH 09/72] Created ReadTheDocs requirements file --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..787317a2e --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +sphinx-copybutton From 6aef69cc81aa9ca151d316a3a878380697541820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jasper=20J=C3=BCrgensen?= Date: Thu, 4 Nov 2021 16:16:16 +0000 Subject: [PATCH 10/72] Update dependencies - attrs from 20.3.0 to 21.2.0 - jinja2 from 2.11.3 to 3.0.2 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 60bcc92ee..1cd5f0e54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ aiohttp==3.7.4 aiohttp-socks==0.5.5 arabic-reshaper==2.1.1 async-timeout==3.0.1 -attrs==20.3.0 +attrs==21.2.0 certifi==2020.12.5 chardet==3.0.4 colorama==0.4.4 @@ -11,7 +11,7 @@ future==0.18.2 future-annotations==1.0.0 html5lib==1.1 idna==2.10 -Jinja2==2.11.3 +Jinja2==3.0.2 lxml==4.6.3 MarkupSafe==1.1.1 mock==4.0.2 From fa3225a7cfe440eecb110a94f380c446cdbe515b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jasper=20J=C3=BCrgensen?= Date: Thu, 4 Nov 2021 21:43:08 +0100 Subject: [PATCH 11/72] Update MarkupSafe to 2.0.1 Fix dependency conflict with jinja2==3.0.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1cd5f0e54..b0d5d2818 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ html5lib==1.1 idna==2.10 Jinja2==3.0.2 lxml==4.6.3 -MarkupSafe==1.1.1 +MarkupSafe==2.0.1 mock==4.0.2 multidict==5.1.0 pycountry==20.7.3 From 1a0db9032de9b165ea2f777798eebbde3bf2670e Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Thu, 4 Nov 2021 23:48:34 +0300 Subject: [PATCH 12/72] Added link to documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 716afbcf7..19e3606f3 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Currently supported more than 2000 sites ([full list](https://raw.githubusercont * Censorship and captcha detection * Requests retries -See full description of Maigret features [in the Wiki](https://github.com/soxoj/maigret/wiki/Features). +See full description of Maigret features [in the documentation](https://maigret.readthedocs.io/en/latest/features.html). ## Installation From dc8fdc25f5a628761bd0f67277c5caaaf7a7257c Mon Sep 17 00:00:00 2001 From: Soxoj Date: Fri, 5 Nov 2021 11:57:51 +0300 Subject: [PATCH 13/72] Upgraded base docker image --- Dockerfile | 2 +- maigret/resources/data.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 38e9dd53c..1ff6424fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.9-slim MAINTAINER Soxoj WORKDIR /app RUN pip install --upgrade pip diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 5a4b51ab3..acd38cb9b 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -13078,7 +13078,7 @@ "us" ], "headers": { - "authorization": "Bearer BQB8QPkkvz_PhWGy4sSY4ijssYjumEHJgJJBFu3VX2Sm4XIoT9jp0eFZrYL3TayY4QZGHmMiz3BCPLcAth4" + "authorization": "Bearer BQAE_L1qFR4DChej3IyXOV0Qvon6anKopqdcnvO6TjZOBa09RuXRGSy29RLO2Pt9DoQezHJCYd6C7a-hzk8" }, "errors": { "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn" @@ -14913,7 +14913,7 @@ "video" ], "headers": { - "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzU2OTI0NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.KZHo96wUe5__rTqZQqAWiJKPKOy2-sjyxRjhOuuhyEc" + "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzYwNjY5ODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.FI1YY8XtBtG4YJsPoV3V-ykYRNcesKI5EymKj1QBRtg" }, "activation": { "url": "https://vimeo.com/_rv/viewer", From f81a500d723e95971872da4e2f80f7324683bf36 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Fri, 5 Nov 2021 13:40:21 +0300 Subject: [PATCH 14/72] Added cascade settings loading from ~/.maigret/settings.json and ./settings.json --- .gitignore | 5 ++- Makefile | 2 +- maigret/maigret.py | 36 ++++++++++--------- maigret/resources/data.json | 4 +-- maigret/resources/settings.json | 4 ++- maigret/settings.py | 61 ++++++++++++++++++++++++--------- tests/conftest.py | 6 +++- tests/test_sites.py | 1 - 8 files changed, 80 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 3be27da9d..862ac39b4 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,7 @@ src/ .coverage dist/ htmlcov/ -/test_* \ No newline at end of file +/test_* + +# Maigret files +settings.json diff --git a/Makefile b/Makefile index 903454349..fa161a680 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ lint: flake8 --count --select=E9,F63,F7,F82 --show-source --statistics ${LINT_FILES} maigret.py @echo 'warning' - flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=E731,W503 ${LINT_FILES} maigret.py + flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=E731,W503,E501 ${LINT_FILES} maigret.py @echo 'mypy' mypy ${LINT_FILES} diff --git a/maigret/maigret.py b/maigret/maigret.py index 27a7919ea..515fffa85 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -8,6 +8,7 @@ import platform from argparse import ArgumentParser, RawDescriptionHelpFormatter from typing import List, Tuple +import os.path as path from socid_extractor import extract, parse @@ -111,7 +112,7 @@ def extract_ids_from_results(results: QueryResultWrapper, db: MaigretDatabase) - return ids_results -def setup_arguments_parser(): +def setup_arguments_parser(settings: Settings): from aiohttp import __version__ as aiohttp_version from requests import __version__ as requests_version from socid_extractor import __version__ as socid_version @@ -159,7 +160,7 @@ def setup_arguments_parser(): action="store", type=int, metavar='RETRIES', - default=1, + default=settings.retries_count, help="Attempts to restart temporarily failed requests.", ) parser.add_argument( @@ -453,9 +454,6 @@ def setup_arguments_parser(): async def main(): - arg_parser = setup_arguments_parser() - args = arg_parser.parse_args() - # Logging log_level = logging.ERROR logging.basicConfig( @@ -463,15 +461,27 @@ async def main(): datefmt='%H:%M:%S', level=log_level, ) + logger = logging.getLogger('maigret') + logger.setLevel(log_level) + # Load settings + settings = Settings() + settings_loaded, err = settings.load() + + if not settings_loaded: + logger.error(err) + sys.exit(3) + + arg_parser = setup_arguments_parser(settings) + args = arg_parser.parse_args() + + # Re-set loggging level based on args if args.debug: log_level = logging.DEBUG elif args.info: log_level = logging.INFO elif args.verbose: log_level = logging.WARNING - - logger = logging.getLogger('maigret') logger.setLevel(log_level) # Usernames initial list @@ -497,15 +507,9 @@ async def main(): if args.tags: args.tags = list(set(str(args.tags).split(','))) - settings = Settings( - os.path.join( - os.path.dirname(os.path.realpath(__file__)), "resources/settings.json" - ) - ) - if args.db_file is None: - args.db_file = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "resources/data.json" + args.db_file = path.join( + path.dirname(path.realpath(__file__)), "resources/data.json" ) if args.top_sites == 0 or args.all_sites: @@ -568,7 +572,7 @@ async def main(): os.makedirs(args.folderoutput, exist_ok=True) # Define one report filename template - report_filepath_tpl = os.path.join(args.folderoutput, 'report_{username}{postfix}') + report_filepath_tpl = path.join(args.folderoutput, 'report_{username}{postfix}') if usernames == {}: # magic params to exit after init diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 5a4b51ab3..cfa27066c 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -13078,7 +13078,7 @@ "us" ], "headers": { - "authorization": "Bearer BQB8QPkkvz_PhWGy4sSY4ijssYjumEHJgJJBFu3VX2Sm4XIoT9jp0eFZrYL3TayY4QZGHmMiz3BCPLcAth4" + "authorization": "Bearer BQCJhizJUIWlnA6-hlMrHeTsb0jCi067H73LzQ1bWk8Iz1tQ6DIE6WyvbJ2x1bapJhZ_MLNiQbTp-dCxz7g" }, "errors": { "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn" @@ -14913,7 +14913,7 @@ "video" ], "headers": { - "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzU2OTI0NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.KZHo96wUe5__rTqZQqAWiJKPKOy2-sjyxRjhOuuhyEc" + "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzYxMDg1NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.kqIzH7O99ov6IrTfeQoA7NNaSZe_7czUgrwKxHnBs3o" }, "activation": { "url": "https://vimeo.com/_rv/viewer", diff --git a/maigret/resources/settings.json b/maigret/resources/settings.json index 1dbe06ec5..e3958554e 100644 --- a/maigret/resources/settings.json +++ b/maigret/resources/settings.json @@ -13,5 +13,7 @@ "e-mail" ], "supposed_usernames": [ - "alex", "god", "admin", "red", "blue", "john"] + "alex", "god", "admin", "red", "blue", "john" + ], + "retries_count": 1 } \ No newline at end of file diff --git a/maigret/settings.py b/maigret/settings.py index 730c2ef14..0f6fd77c1 100644 --- a/maigret/settings.py +++ b/maigret/settings.py @@ -1,28 +1,57 @@ +import os +import os.path as path import json +SETTINGS_FILES_PATHS = [ + path.join(path.dirname(path.realpath(__file__)), "resources/settings.json"), + '~/.maigret/settings.json', + path.join(os.getcwd(), 'settings.json'), +] + class Settings: + # main maigret setting + retries_count: int + + # submit mode settings presence_strings: list supposed_usernames: list - def __init__(self, filename): - data = {} + def __init__(self): + pass + + def load(self, paths=None): + was_inited = False + + if not paths: + paths = SETTINGS_FILES_PATHS - try: - with open(filename, "r", encoding="utf-8") as file: - try: + for filename in paths: + data = {} + + try: + with open(filename, "r", encoding="utf-8") as file: data = json.load(file) - except Exception as error: - raise ValueError( - f"Problem with parsing json contents of " - f"settings file '{filename}': {str(error)}." - ) - except FileNotFoundError as error: - raise FileNotFoundError( - f"Problem while attempting to access settings file '{filename}'." - ) from error - - self.__dict__.update(data) + except FileNotFoundError: + # treast as a normal situation + pass + except Exception as error: + return False, ValueError( + f"Problem with parsing json contents of " + f"settings file '{filename}': {str(error)}." + ) + + self.__dict__.update(data) + import logging + + logging.error(data) + if data: + was_inited = True + + return ( + was_inited, + f'None of the default settings files found: {", ".join(paths)}', + ) @property def json(self): diff --git a/tests/conftest.py b/tests/conftest.py index 2210eaa5f..4bbc8c00d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,10 +7,12 @@ from maigret.sites import MaigretDatabase from maigret.maigret import setup_arguments_parser +from maigret.settings import Settings CUR_PATH = os.path.dirname(os.path.realpath(__file__)) JSON_FILE = os.path.join(CUR_PATH, '../maigret/resources/data.json') +SETTINGS_FILE = os.path.join(CUR_PATH, '../maigret/resources/settings.json') TEST_JSON_FILE = os.path.join(CUR_PATH, 'db.json') LOCAL_TEST_JSON_FILE = os.path.join(CUR_PATH, 'local.json') empty_mark = Mark('', (), {}) @@ -59,7 +61,9 @@ def reports_autoclean(): @pytest.fixture(scope='session') def argparser(): - return setup_arguments_parser() + settings = Settings() + settings.load([SETTINGS_FILE]) + return setup_arguments_parser(settings) @pytest.fixture(scope="session") diff --git a/tests/test_sites.py b/tests/test_sites.py index f49492291..23610c63c 100644 --- a/tests/test_sites.py +++ b/tests/test_sites.py @@ -1,6 +1,5 @@ """Maigret Database test functions""" from maigret.sites import MaigretDatabase, MaigretSite -from maigret.utils import URLMatcher EXAMPLE_DB = { 'engines': { From 59f573e7546dda299a089535871944b74e346af0 Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Fri, 5 Nov 2021 13:57:00 +0300 Subject: [PATCH 15/72] Run CodeQL only aflter merge and each Saturday --- .github/workflows/codeql-analysis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3c73d6bda..745c89cd3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -14,11 +14,8 @@ name: "CodeQL" on: push: branches: [ main ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ main ] schedule: - - cron: '23 6 * * 1' + - cron: '23 6 * * 6' jobs: analyze: From 7a362406d5abaf3e83b5b74b83223e14ebe60de4 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sat, 6 Nov 2021 08:22:23 +0300 Subject: [PATCH 16/72] Documentation and settings improved --- README.md | 2 +- docs/source/command-line-options.rst | 5 +++++ docs/source/index.rst | 1 + docs/source/settings.rst | 26 +++++++++++++++++++++++ maigret/maigret.py | 31 ++++++++++++++-------------- maigret/resources/data.json | 4 ++-- maigret/resources/settings.json | 7 ++++++- maigret/settings.py | 5 +++++ tests/test_cli.py | 2 +- 9 files changed, 62 insertions(+), 21 deletions(-) create mode 100644 docs/source/settings.rst diff --git a/README.md b/README.md index 19e3606f3..06255149f 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ **Maigret** collect a dossier on a person **by username only**, checking for accounts on a huge number of sites and gathering all the available information from web pages. No API keys required. Maigret is an easy-to-use and powerful fork of [Sherlock](https://github.com/sherlock-project/sherlock). -Currently supported more than 2000 sites ([full list](https://raw.githubusercontent.com/soxoj/maigret/main/sites.md)), search is launched against 500 popular sites in descending order of popularity by default. Also supported checking of Tor sites, I2P sites, and domains (via DNS resolving). +Currently supported more than 2500 sites ([full list](https://github.com/soxoj/maigret/blob/main/sites.md)), search is launched against 500 popular sites in descending order of popularity by default. Also supported checking of Tor sites, I2P sites, and domains (via DNS resolving). ## Main features diff --git a/docs/source/command-line-options.rst b/docs/source/command-line-options.rst index 44dcd8f62..dea15d349 100644 --- a/docs/source/command-line-options.rst +++ b/docs/source/command-line-options.rst @@ -23,6 +23,11 @@ extracted username and ids. :doc:`Examples `. Main options ------------ +Options are also configurable through settings files. See +:doc:`settings section ` +`settings.json ` +for the list of currently supported options. + ``--tags`` - Filter sites for searching by tags: sites categories and two-letter country codes. E.g. photo, dating, sport; jp, us, global. Multiple tags can be associated with one site. **Warning: tags markup is diff --git a/docs/source/index.rst b/docs/source/index.rst index ec6bda64d..273e4cfae 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -27,3 +27,4 @@ You may be interested in: supported-identifier-types tags usage-examples + settings diff --git a/docs/source/settings.rst b/docs/source/settings.rst new file mode 100644 index 000000000..4ef826bc7 --- /dev/null +++ b/docs/source/settings.rst @@ -0,0 +1,26 @@ +.. _settings: + +Settings +============== + +Options are also configurable through settings files. See +`settings.json ` +for the list of currently supported options. + +After start Maigret tries to load configuration from the following sources in exactly the same order: + +.. code-block:: console + + # relative path, based on installed package path + resources/settings.json + + # absolute path, configuration file in home directory + ~/.maigret/settings.json + + # relative path, based on current working directory + settings.json + +Missing any of these files is not an error. +If the next settings file contains already known option, +this option will be rewrited. So it is possible to make +custom configuration for different users and directories. diff --git a/maigret/maigret.py b/maigret/maigret.py index 515fffa85..0d04f4c8e 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -129,7 +129,9 @@ def setup_arguments_parser(settings: Settings): parser = ArgumentParser( formatter_class=RawDescriptionHelpFormatter, - description=f"Maigret v{__version__}", + description=f"Maigret v{__version__}\n" + "Documentation: https://maigret.readthedocs.io/\n" + "All settings are also configurable through files, see docs.", ) parser.add_argument( "username", @@ -149,9 +151,9 @@ def setup_arguments_parser(settings: Settings): metavar='TIMEOUT', dest="timeout", type=timeout_check, - default=30, - help="Time in seconds to wait for response to requests. " - "Default timeout of 30.0s. " + default=settings.timeout, + help="Time in seconds to wait for response to requests " + f"(default {settings.timeout}s). " "A longer timeout will be more likely to get results from slow sites. " "On the other hand, this may cause a long delay to gather all results. ", ) @@ -169,21 +171,21 @@ def setup_arguments_parser(settings: Settings): action="store", type=int, dest="connections", - default=100, + default=settings.max_connections, help="Allowed number of concurrent connections.", ) parser.add_argument( "--no-recursion", action="store_true", dest="disable_recursive_search", - default=False, + default=(not settings.recursive_search), help="Disable recursive search by additional data extracted from pages.", ) parser.add_argument( "--no-extracting", action="store_true", dest="disable_extracting", - default=False, + default=(not settings.info_extracting), help="Disable parsing pages for additional data and other usernames.", ) parser.add_argument( @@ -197,7 +199,7 @@ def setup_arguments_parser(settings: Settings): "--db", metavar="DB_FILE", dest="db_file", - default=None, + default=settings.sites_db_path, help="Load Maigret database from a JSON file or HTTP web resource.", ) parser.add_argument( @@ -507,10 +509,7 @@ async def main(): if args.tags: args.tags = list(set(str(args.tags).split(','))) - if args.db_file is None: - args.db_file = path.join( - path.dirname(path.realpath(__file__)), "resources/data.json" - ) + db_file = path.join(path.dirname(path.realpath(__file__)), args.db_file) if args.top_sites == 0 or args.all_sites: args.top_sites = sys.maxsize @@ -525,7 +524,7 @@ async def main(): ) # Create object with all information about sites we are aware of. - db = MaigretDatabase().load_from_path(args.db_file) + db = MaigretDatabase().load_from_path(db_file) get_top_sites_for_id = lambda x: db.ranked_sites_dict( top=args.top_sites, tags=args.tags, @@ -540,7 +539,7 @@ async def main(): submitter = Submitter(db=db, logger=logger, settings=settings) is_submitted = await submitter.dialog(args.new_site_to_submit, args.cookie_file) if is_submitted: - db.save_to_file(args.db_file) + db.save_to_file(db_file) # Database self-checking if args.self_check: @@ -558,7 +557,7 @@ async def main(): 'y', '', ): - db.save_to_file(args.db_file) + db.save_to_file(db_file) print('Database was successfully updated.') else: print('Updates will be applied only for current search session.') @@ -708,7 +707,7 @@ async def main(): print(text_report) # update database - db.save_to_file(args.db_file) + db.save_to_file(db_file) def run(): diff --git a/maigret/resources/data.json b/maigret/resources/data.json index cfa27066c..5a4b51ab3 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -13078,7 +13078,7 @@ "us" ], "headers": { - "authorization": "Bearer BQCJhizJUIWlnA6-hlMrHeTsb0jCi067H73LzQ1bWk8Iz1tQ6DIE6WyvbJ2x1bapJhZ_MLNiQbTp-dCxz7g" + "authorization": "Bearer BQB8QPkkvz_PhWGy4sSY4ijssYjumEHJgJJBFu3VX2Sm4XIoT9jp0eFZrYL3TayY4QZGHmMiz3BCPLcAth4" }, "errors": { "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn" @@ -14913,7 +14913,7 @@ "video" ], "headers": { - "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzYxMDg1NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.kqIzH7O99ov6IrTfeQoA7NNaSZe_7czUgrwKxHnBs3o" + "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzU2OTI0NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.KZHo96wUe5__rTqZQqAWiJKPKOy2-sjyxRjhOuuhyEc" }, "activation": { "url": "https://vimeo.com/_rv/viewer", diff --git a/maigret/resources/settings.json b/maigret/resources/settings.json index e3958554e..300af3344 100644 --- a/maigret/resources/settings.json +++ b/maigret/resources/settings.json @@ -15,5 +15,10 @@ "supposed_usernames": [ "alex", "god", "admin", "red", "blue", "john" ], - "retries_count": 1 + "retries_count": 1, + "sites_db_path": "resources/data.json", + "timeout": 30, + "max_connections": 100, + "recursive_search": true, + "info_extracting": true } \ No newline at end of file diff --git a/maigret/settings.py b/maigret/settings.py index 0f6fd77c1..b0fca08eb 100644 --- a/maigret/settings.py +++ b/maigret/settings.py @@ -12,6 +12,11 @@ class Settings: # main maigret setting retries_count: int + sites_db_path: str + timeout: int + max_connections: int + recursive_search: bool + info_extracting: bool # submit mode settings presence_strings: list diff --git a/tests/test_cli.py b/tests/test_cli.py index 5a3f53e08..8d677f1b4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,7 +7,7 @@ 'connections': 100, 'cookie_file': None, 'csv': False, - 'db_file': None, + 'db_file': 'resources/data.json', 'debug': False, 'disable_extracting': False, 'disable_recursive_search': False, From 267d9e505bb11a2675919b3a83a13dfc942a501e Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sat, 6 Nov 2021 14:29:56 +0300 Subject: [PATCH 17/72] New config options added --- docs/source/command-line-options.rst | 4 ++++ maigret/maigret.py | 12 ++++++------ maigret/resources/settings.json | 13 ++++++++++++- maigret/settings.py | 12 ++++++++++++ 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/docs/source/command-line-options.rst b/docs/source/command-line-options.rst index 44dcd8f62..c6f4bcdc7 100644 --- a/docs/source/command-line-options.rst +++ b/docs/source/command-line-options.rst @@ -23,6 +23,10 @@ extracted username and ids. :doc:`Examples `. Main options ------------ +======= +Options are also configurable through settings files, see +:doc:`settings section `. + ``--tags`` - Filter sites for searching by tags: sites categories and two-letter country codes. E.g. photo, dating, sport; jp, us, global. Multiple tags can be associated with one site. **Warning: tags markup is diff --git a/maigret/maigret.py b/maigret/maigret.py index 515fffa85..ae821140d 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -204,7 +204,7 @@ def setup_arguments_parser(settings: Settings): "--cookies-jar-file", metavar="COOKIE_FILE", dest="cookie_file", - default=None, + default=settings.cookie_jar_file, help="File with cookies.", ) parser.add_argument( @@ -212,7 +212,7 @@ def setup_arguments_parser(settings: Settings): action="append", metavar='IGNORED_IDS', dest="ignore_ids_list", - default=[], + default=settings.ignore_ids_list, help="Do not make search by the specified username or other ids.", ) # reports options @@ -220,7 +220,7 @@ def setup_arguments_parser(settings: Settings): "--folderoutput", "-fo", dest="folderoutput", - default="reports", + default=settings.reports_path, metavar="PATH", help="If using multiple usernames, the output of the results will be saved to this folder.", ) @@ -230,21 +230,21 @@ def setup_arguments_parser(settings: Settings): metavar='PROXY_URL', action="store", dest="proxy", - default=None, + default=settings.proxy_url, help="Make requests over a proxy. e.g. socks5://127.0.0.1:1080", ) parser.add_argument( "--tor-proxy", metavar='TOR_PROXY_URL', action="store", - default='socks5://127.0.0.1:9050', + default=settings.tor_proxy_url, help="Specify URL of your Tor gateway. Default is socks5://127.0.0.1:9050", ) parser.add_argument( "--i2p-proxy", metavar='I2P_PROXY_URL', action="store", - default='http://127.0.0.1:4444', + default=settings.i2p_proxy_url, help="Specify URL of your I2P gateway. Default is http://127.0.0.1:4444", ) parser.add_argument( diff --git a/maigret/resources/settings.json b/maigret/resources/settings.json index e3958554e..88974b27a 100644 --- a/maigret/resources/settings.json +++ b/maigret/resources/settings.json @@ -15,5 +15,16 @@ "supposed_usernames": [ "alex", "god", "admin", "red", "blue", "john" ], - "retries_count": 1 + "retries_count": 1, + "sites_db_path": "resources/data.json", + "timeout": 30, + "max_connections": 100, + "recursive_search": true, + "info_extracting": true, + "cookie_jar_file": null, + "ignore_ids_list": [], + "reports_path": "reports", + "proxy_url": null, + "tor_proxy_url": "socks5://127.0.0.1:9050", + "i2p_proxy_url": "http://127.0.0.1:4444" } \ No newline at end of file diff --git a/maigret/settings.py b/maigret/settings.py index 0f6fd77c1..1ab2fd793 100644 --- a/maigret/settings.py +++ b/maigret/settings.py @@ -1,6 +1,7 @@ import os import os.path as path import json +from typing import List SETTINGS_FILES_PATHS = [ path.join(path.dirname(path.realpath(__file__)), "resources/settings.json"), @@ -12,6 +13,17 @@ class Settings: # main maigret setting retries_count: int + sites_db_path: str + timeout: int + max_connections: int + recursive_search: bool + info_extracting: bool + cookie_jar_file: str + ignore_ids_list: List + reports_path: str + proxy_url: str + tor_proxy_url: str + i2p_proxy_url: str # submit mode settings presence_strings: list From 646265791a184fc34c569bf45fe29f93c5afc9f9 Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Sun, 7 Nov 2021 18:14:48 +0300 Subject: [PATCH 18/72] Added export of cli entrypoint --- maigret/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/maigret/__init__.py b/maigret/__init__.py index 8aeddeb99..3cf3f1101 100644 --- a/maigret/__init__.py +++ b/maigret/__init__.py @@ -8,5 +8,6 @@ from .__version__ import __version__ from .checking import maigret as search +from .maigret import main as cli from .sites import MaigretEngine, MaigretSite, MaigretDatabase from .notify import QueryNotifyPrint as Notifier From 1b9ce3bac79de25f93c5ecc5e403dea9f3478574 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 7 Nov 2021 18:52:56 +0300 Subject: [PATCH 19/72] Removed redundant logging --- docs/source/settings.rst | 2 +- maigret/settings.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 4ef826bc7..bbe789a4e 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -4,7 +4,7 @@ Settings ============== Options are also configurable through settings files. See -`settings.json ` +`settings JSON file `_ for the list of currently supported options. After start Maigret tries to load configuration from the following sources in exactly the same order: diff --git a/maigret/settings.py b/maigret/settings.py index 1ab2fd793..57fe0de72 100644 --- a/maigret/settings.py +++ b/maigret/settings.py @@ -54,9 +54,6 @@ def load(self, paths=None): ) self.__dict__.update(data) - import logging - - logging.error(data) if data: was_inited = True From 1290a9863fc6cc2b9298e235c428e8b88fca139d Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Sun, 7 Nov 2021 20:12:30 +0300 Subject: [PATCH 20/72] PyInstaller workflow (#206) * PyInstaller workflow --- .github/workflows/pyinstaller.yml | 24 +++++++++++++ pyinstaller/maigret_standalone.py | 7 ++++ pyinstaller/maigret_standalone.spec | 55 +++++++++++++++++++++++++++++ pyinstaller/requirements.txt | 5 +++ 4 files changed, 91 insertions(+) create mode 100644 .github/workflows/pyinstaller.yml create mode 100755 pyinstaller/maigret_standalone.py create mode 100644 pyinstaller/maigret_standalone.spec create mode 100644 pyinstaller/requirements.txt diff --git a/.github/workflows/pyinstaller.yml b/.github/workflows/pyinstaller.yml new file mode 100644 index 000000000..b4fde8ce5 --- /dev/null +++ b/.github/workflows/pyinstaller.yml @@ -0,0 +1,24 @@ +name: Package exe with PyInstaller - Windows + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: PyInstaller Windows + uses: JackMcKew/pyinstaller-action-windows@main + with: + path: pyinstaller + + - uses: actions/upload-artifact@v2 + with: + name: maigret_standalone + path: pyinstaller/dist/windows # or path/to/artifact diff --git a/pyinstaller/maigret_standalone.py b/pyinstaller/maigret_standalone.py new file mode 100755 index 000000000..5b83b4477 --- /dev/null +++ b/pyinstaller/maigret_standalone.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +import asyncio + +import maigret + +if __name__ == "__main__": + asyncio.run(maigret.cli()) \ No newline at end of file diff --git a/pyinstaller/maigret_standalone.spec b/pyinstaller/maigret_standalone.spec new file mode 100644 index 000000000..a5341efec --- /dev/null +++ b/pyinstaller/maigret_standalone.spec @@ -0,0 +1,55 @@ +# -*- mode: python ; coding: utf-8 -*- +from PyInstaller.utils.hooks import collect_all + +datas = [] +binaries = [] +hiddenimports = [] + +full_import_modules = ['maigret', 'socid_extractor', 'arabic_reshaper', 'pyvis', 'reportlab'] + +for module in full_import_modules: + tmp_ret = collect_all(module) + datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] + +hiddenimports += ['PySocks', 'beautifulsoup4', 'python-dateutil', + 'future-annotations', 'six', 'python-bidi', + 'typing-extensions', 'attrs', 'torrequest'] + +block_cipher = None + + +a = Analysis(['maigret_standalone.py'], + pathex=[], + binaries=binaries, + datas=datas, + hiddenimports=hiddenimports, + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) + +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) + +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='maigret_standalone', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None ) diff --git a/pyinstaller/requirements.txt b/pyinstaller/requirements.txt new file mode 100644 index 000000000..f03eae09e --- /dev/null +++ b/pyinstaller/requirements.txt @@ -0,0 +1,5 @@ +maigret @ https://github.com/soxoj/maigret/archive/refs/heads/main.zip +pefile==2019.4.18 +psutil==5.7.0 +pyinstaller @ https://github.com/pyinstaller/pyinstaller/archive/develop.zip +pywin32-ctypes==0.2.0 \ No newline at end of file From 3dc8ae1f414a65525c53c823ee5b234584cfc1de Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Sun, 7 Nov 2021 22:06:57 +0300 Subject: [PATCH 21/72] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 06255149f..f5404eb2f 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,9 @@ See full description of Maigret features [in the documentation](https://maigret. ## Installation Maigret can be installed using pip, Docker, or simply can be launched from the cloned repo. + +Standalone EXE-binaries for Windows are located in [Releases section](https://github.com/soxoj/maigret/releases) of GitHub repository. + Also you can run Maigret using cloud shells and Jupyter notebooks (see buttons below). [![Open in Cloud Shell](https://user-images.githubusercontent.com/27065646/92304704-8d146d80-ef80-11ea-8c29-0deaabb1c702.png)](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/soxoj/maigret&tutorial=README.md) From 6688479c1cf97828f02f850ef643884d835b6780 Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Mon, 8 Nov 2021 19:52:53 +0300 Subject: [PATCH 22/72] Create bug.md (#213) --- .github/ISSUE_TEMPLATE/bug.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug.md diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md new file mode 100644 index 000000000..3941ad0e1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -0,0 +1,24 @@ +--- +name: Maigret bug report +about: I want to report a bug in Maigret functionality +title: '' +labels: bug +assignees: soxoj + +--- + +## Checklist + +- [ ] I'm reporting a bug in Maigret functionality +- [ ] I've checked for similar bug reports including closed ones +- [ ] I've checked for pull requests that attempt to fix this bug + +## Description + +Info about Maigret version you are running and environment (`--version`, operation system, ISP provuder): + + +How to reproduce this bug (commandline options / conditions): + + + From c0f2a550f5bb9482bf96c9f1266b0b8c8d1b7d65 Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Tue, 9 Nov 2021 00:21:19 +0300 Subject: [PATCH 23/72] Fixed path and names of report files (#216) --- maigret/maigret.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/maigret/maigret.py b/maigret/maigret.py index 6383563ee..dd7c682cb 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -567,11 +567,13 @@ async def main(): if args.stats: print(db.get_db_stats(db.sites_dict)) + report_dir = path.join(os.getcwd(), args.folderoutput) + # Make reports folder is not exists - os.makedirs(args.folderoutput, exist_ok=True) + os.makedirs(report_dir, exist_ok=True) # Define one report filename template - report_filepath_tpl = path.join(args.folderoutput, 'report_{username}{postfix}') + report_filepath_tpl = path.join(report_dir, 'report_{username}{postfix}') if usernames == {}: # magic params to exit after init @@ -687,7 +689,7 @@ async def main(): username = report_context['username'] if args.html: - filename = report_filepath_tpl.format(username=username, postfix='.html') + filename = report_filepath_tpl.format(username=username, postfix='_plain.html') save_html_report(filename, report_context) query_notify.warning(f'HTML report on all usernames saved in {filename}') @@ -697,7 +699,7 @@ async def main(): query_notify.warning(f'PDF report on all usernames saved in {filename}') if args.graph: - filename = report_filepath_tpl.format(username=username, postfix='.html') + filename = report_filepath_tpl.format(username=username, postfix='_graph.html') save_graph_report(filename, general_results, db) query_notify.warning(f'Graph report on all usernames saved in {filename}') From 7decbce08da7fb821ecc7cda00ac6a64b1b7d5a9 Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Tue, 9 Nov 2021 02:33:31 +0300 Subject: [PATCH 24/72] Box drawing logic improved, added new settings (#217) --- maigret/checking.py | 1 + maigret/maigret.py | 44 ++++++++++++++++++--------------- maigret/resources/data.json | 4 +-- maigret/resources/settings.json | 20 ++++++++++++++- maigret/settings.py | 18 ++++++++++++++ maigret/utils.py | 12 +++++++-- tests/test_utils.py | 26 +++++++++---------- 7 files changed, 87 insertions(+), 38 deletions(-) diff --git a/maigret/checking.py b/maigret/checking.py index 08c42d720..603eeee39 100644 --- a/maigret/checking.py +++ b/maigret/checking.py @@ -28,6 +28,7 @@ AsyncioSimpleExecutor, AsyncioProgressbarQueueExecutor, ) + from .result import QueryResult, QueryStatus from .sites import MaigretDatabase, MaigretSite from .types import QueryOptions, QueryResultWrapper diff --git a/maigret/maigret.py b/maigret/maigret.py index dd7c682cb..3a5dac011 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -252,7 +252,7 @@ def setup_arguments_parser(settings: Settings): parser.add_argument( "--with-domains", action="store_true", - default=False, + default=settings.domain_search, help="Enable (experimental) feature of checking domains on usernames.", ) @@ -264,13 +264,13 @@ def setup_arguments_parser(settings: Settings): "--all-sites", action="store_true", dest="all_sites", - default=False, + default=settings.scan_all_sites, help="Use all sites for scan.", ) filter_group.add_argument( "--top-sites", action="store", - default=500, + default=settings.top_sites_count, metavar="N", type=int, help="Count of sites for scan ranked by Alexa Top (default: 500).", @@ -283,13 +283,13 @@ def setup_arguments_parser(settings: Settings): action="append", metavar='SITE_NAME', dest="site_list", - default=[], + default=settings.scan_sites_list, help="Limit analysis to just the specified sites (multiple option).", ) filter_group.add_argument( "--use-disabled-sites", action="store_true", - default=False, + default=settings.scan_disabled_sites, help="Use disabled sites to search (may cause many false positives).", ) @@ -316,7 +316,7 @@ def setup_arguments_parser(settings: Settings): modes_group.add_argument( "--self-check", action="store_true", - default=False, + default=settings.self_check_enabled, help="Do self check for sites and database and disable non-working ones.", ) modes_group.add_argument( @@ -333,14 +333,14 @@ def setup_arguments_parser(settings: Settings): "--print-not-found", action="store_true", dest="print_not_found", - default=False, + default=settings.print_not_found, help="Print sites where the username was not found.", ) output_group.add_argument( "--print-errors", action="store_true", dest="print_check_errors", - default=False, + default=settings.print_check_errors, help="Print errors messages: connection, captcha, site country ban, etc.", ) output_group.add_argument( @@ -372,14 +372,14 @@ def setup_arguments_parser(settings: Settings): "--no-color", action="store_true", dest="no_color", - default=False, + default=(not settings.colored_print), help="Don't color terminal output", ) output_group.add_argument( "--no-progressbar", action="store_true", dest="no_progressbar", - default=False, + default=(not settings.show_progressbar), help="Don't show progressbar.", ) @@ -391,7 +391,7 @@ def setup_arguments_parser(settings: Settings): "--txt", action="store_true", dest="txt", - default=False, + default=settings.txt_report, help="Create a TXT report (one report per username).", ) report_group.add_argument( @@ -399,7 +399,7 @@ def setup_arguments_parser(settings: Settings): "--csv", action="store_true", dest="csv", - default=False, + default=settings.csv_report, help="Create a CSV report (one report per username).", ) report_group.add_argument( @@ -407,7 +407,7 @@ def setup_arguments_parser(settings: Settings): "--html", action="store_true", dest="html", - default=False, + default=settings.html_report, help="Create an HTML report file (general report on all usernames).", ) report_group.add_argument( @@ -415,7 +415,7 @@ def setup_arguments_parser(settings: Settings): "--xmind", action="store_true", dest="xmind", - default=False, + default=settings.xmind_report, help="Generate an XMind 8 mindmap report (one report per username).", ) report_group.add_argument( @@ -423,7 +423,7 @@ def setup_arguments_parser(settings: Settings): "--pdf", action="store_true", dest="pdf", - default=False, + default=settings.pdf_report, help="Generate a PDF report (general report on all usernames).", ) report_group.add_argument( @@ -431,7 +431,7 @@ def setup_arguments_parser(settings: Settings): "--graph", action="store_true", dest="graph", - default=False, + default=settings.graph_report, help="Generate a graph report (general report on all usernames).", ) report_group.add_argument( @@ -440,7 +440,7 @@ def setup_arguments_parser(settings: Settings): action="store", metavar='TYPE', dest="json", - default='', + default=settings.json_report_type, choices=SUPPORTED_JSON_REPORT_FORMATS, help=f"Generate a JSON report of specific type: {', '.join(SUPPORTED_JSON_REPORT_FORMATS)}" " (one report per username).", @@ -448,7 +448,7 @@ def setup_arguments_parser(settings: Settings): parser.add_argument( "--reports-sorting", - default='default', + default=settings.report_sorting, choices=('default', 'data'), help="Method of results sorting in reports (default: in order of getting the result)", ) @@ -689,7 +689,9 @@ async def main(): username = report_context['username'] if args.html: - filename = report_filepath_tpl.format(username=username, postfix='_plain.html') + filename = report_filepath_tpl.format( + username=username, postfix='_plain.html' + ) save_html_report(filename, report_context) query_notify.warning(f'HTML report on all usernames saved in {filename}') @@ -699,7 +701,9 @@ async def main(): query_notify.warning(f'PDF report on all usernames saved in {filename}') if args.graph: - filename = report_filepath_tpl.format(username=username, postfix='_graph.html') + filename = report_filepath_tpl.format( + username=username, postfix='_graph.html' + ) save_graph_report(filename, general_results, db) query_notify.warning(f'Graph report on all usernames saved in {filename}') diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 5a4b51ab3..3de56a764 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -13078,7 +13078,7 @@ "us" ], "headers": { - "authorization": "Bearer BQB8QPkkvz_PhWGy4sSY4ijssYjumEHJgJJBFu3VX2Sm4XIoT9jp0eFZrYL3TayY4QZGHmMiz3BCPLcAth4" + "authorization": "Bearer BQB8pjfUq_eeuyy-yvDy5ptoq01QtMbLG0uPW_0t8La7CHd8l4pN3iY4rm5ZBkCEdeWmXyF313-0w9FSp1E" }, "errors": { "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn" @@ -14913,7 +14913,7 @@ "video" ], "headers": { - "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzU2OTI0NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.KZHo96wUe5__rTqZQqAWiJKPKOy2-sjyxRjhOuuhyEc" + "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzY0MDY3NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.7tNEeKdpPtvT3k4G-7neQhO1SniqondSkOQ50SUkuNc" }, "activation": { "url": "https://vimeo.com/_rv/viewer", diff --git a/maigret/resources/settings.json b/maigret/resources/settings.json index 88974b27a..fe0480799 100644 --- a/maigret/resources/settings.json +++ b/maigret/resources/settings.json @@ -26,5 +26,23 @@ "reports_path": "reports", "proxy_url": null, "tor_proxy_url": "socks5://127.0.0.1:9050", - "i2p_proxy_url": "http://127.0.0.1:4444" + "i2p_proxy_url": "http://127.0.0.1:4444", + "domain_search": false, + "scan_all_sites": false, + "top_sites_count": 500, + "scan_disabled_sites": false, + "scan_sites_list": [], + "self_check_enabled": false, + "print_not_found": false, + "print_check_errors": false, + "colored_print": true, + "show_progressbar": true, + "report_sorting": "default", + "json_report_type": "", + "txt_report": false, + "csv_report": false, + "xmind_report": false, + "graph_report": false, + "pdf_report": false, + "html_report": false } \ No newline at end of file diff --git a/maigret/settings.py b/maigret/settings.py index 57fe0de72..31b9ca518 100644 --- a/maigret/settings.py +++ b/maigret/settings.py @@ -24,6 +24,24 @@ class Settings: proxy_url: str tor_proxy_url: str i2p_proxy_url: str + domain_search: bool + scan_all_sites: bool + top_sites_count: int + scan_disabled_sites: bool + scan_sites_list: List + self_check_enabled: bool + print_not_found: bool + print_check_errors: bool + colored_print: bool + show_progressbar: bool + report_sorting: str + json_report_type: str + txt_report: bool + csv_report: bool + xmind_report: bool + pdf_report: bool + html_report: bool + graph_report: bool # submit mode settings presence_strings: list diff --git a/maigret/utils.py b/maigret/utils.py index 296c14036..ab414a026 100644 --- a/maigret/utils.py +++ b/maigret/utils.py @@ -1,3 +1,4 @@ +# coding: utf8 import ast import difflib import re @@ -73,15 +74,22 @@ def ascii_data_display(data: str) -> Any: def get_dict_ascii_tree(items, prepend="", new_line=True): + new_result = b'\xe2\x94\x9c'.decode() + new_line = b'\xe2\x94\x80'.decode() + last_result = b'\xe2\x94\x94'.decode() + skip_result = b'\xe2\x94\x82'.decode() + text = "" for num, item in enumerate(items): - box_symbol = "┣╸" if num != len(items) - 1 else "┗╸" + box_symbol = ( + new_result + new_line if num != len(items) - 1 else last_result + new_line + ) if type(item) == tuple: field_name, field_value = item if field_value.startswith("['"): is_last_item = num == len(items) - 1 - prepend_symbols = " " * 3 if is_last_item else " ┃ " + prepend_symbols = " " * 3 if is_last_item else f" {skip_result} " data = ascii_data_display(field_value) field_value = get_dict_ascii_tree(data, prepend_symbols) text += f"\n{prepend}{box_symbol}{field_name}: {field_value}" diff --git a/tests/test_utils.py b/tests/test_utils.py index 0ff780193..951fe9fd5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -123,19 +123,19 @@ def test_get_dict_ascii_tree(): assert ( ascii_tree == """ - ┣╸uid: dXJpOm5vZGU6VXNlcjoyNjQwMzQxNQ== - ┣╸legacy_id: 26403415 - ┣╸username: alexaimephotographycars - ┣╸name: Alex Aimé - ┣╸links: - ┃ ┗╸ www.instagram.com/street.reality.photography/ - ┣╸created_at: 2018-05-04T10:17:01.000+0000 - ┣╸image: https://drscdn.500px.org/user_avatar/26403415/q%3D85_w%3D300_h%3D300/v2?webp=true&v=2&sig=0235678a4f7b65e007e864033ebfaf5ef6d87fad34f80a8639d985320c20fe3b - ┣╸image_bg: https://drscdn.500px.org/user_cover/26403415/q%3D65_m%3D2048/v2?webp=true&v=1&sig=bea411fb158391a4fdad498874ff17088f91257e59dfb376ff67e3a44c3a4201 - ┣╸website: www.instagram.com/street.reality.photography/ - ┣╸facebook_link: www.instagram.com/street.reality.photography/ - ┣╸instagram_username: Street.Reality.Photography - ┗╸twitter_username: Alexaimephotogr""" + ├─uid: dXJpOm5vZGU6VXNlcjoyNjQwMzQxNQ== + ├─legacy_id: 26403415 + ├─username: alexaimephotographycars + ├─name: Alex Aimé + ├─links: + │ └─ www.instagram.com/street.reality.photography/ + ├─created_at: 2018-05-04T10:17:01.000+0000 + ├─image: https://drscdn.500px.org/user_avatar/26403415/q%3D85_w%3D300_h%3D300/v2?webp=true&v=2&sig=0235678a4f7b65e007e864033ebfaf5ef6d87fad34f80a8639d985320c20fe3b + ├─image_bg: https://drscdn.500px.org/user_cover/26403415/q%3D65_m%3D2048/v2?webp=true&v=1&sig=bea411fb158391a4fdad498874ff17088f91257e59dfb376ff67e3a44c3a4201 + ├─website: www.instagram.com/street.reality.photography/ + ├─facebook_link: www.instagram.com/street.reality.photography/ + ├─instagram_username: Street.Reality.Photography + └─twitter_username: Alexaimephotogr""" ) From cae9bf99ffb1bdb62adb6549624c9766cae03b04 Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Tue, 9 Nov 2021 23:14:42 +0300 Subject: [PATCH 25/72] Fixes for win32 release (#218) --- .github/workflows/pyinstaller.yml | 2 +- pyinstaller/maigret_standalone.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pyinstaller.yml b/.github/workflows/pyinstaller.yml index b4fde8ce5..63ad8d696 100644 --- a/.github/workflows/pyinstaller.yml +++ b/.github/workflows/pyinstaller.yml @@ -20,5 +20,5 @@ jobs: - uses: actions/upload-artifact@v2 with: - name: maigret_standalone + name: maigret_standalone_win32 path: pyinstaller/dist/windows # or path/to/artifact diff --git a/pyinstaller/maigret_standalone.spec b/pyinstaller/maigret_standalone.spec index a5341efec..3152748ce 100644 --- a/pyinstaller/maigret_standalone.spec +++ b/pyinstaller/maigret_standalone.spec @@ -5,7 +5,7 @@ datas = [] binaries = [] hiddenimports = [] -full_import_modules = ['maigret', 'socid_extractor', 'arabic_reshaper', 'pyvis', 'reportlab'] +full_import_modules = ['maigret', 'socid_extractor', 'arabic_reshaper', 'pyvis', 'reportlab.graphics.barcode'] for module in full_import_modules: tmp_ret = collect_all(module) From 10523e98c5ea8b01d137778d3a95cd001eb74a8a Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Wed, 10 Nov 2021 15:52:47 +0300 Subject: [PATCH 26/72] Create dependabot.yml --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..b38df29f4 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" From 2059e69e993e6fbee90624f0ea94027b0ca17e90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 16:12:25 +0300 Subject: [PATCH 27/72] Bump six from 1.15.0 to 1.16.0 (#221) Bumps [six](https://github.com/benjaminp/six) from 1.15.0 to 1.16.0. - [Release notes](https://github.com/benjaminp/six/releases) - [Changelog](https://github.com/benjaminp/six/blob/master/CHANGES) - [Commits](https://github.com/benjaminp/six/compare/1.15.0...1.16.0) --- updated-dependencies: - dependency-name: six dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b0d5d2818..3f30d460a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ python-bidi==0.4.2 python-socks==1.1.2 requests==2.26.0 requests-futures==1.0.0 -six==1.15.0 +six==1.16.0 socid-extractor>=0.0.21 soupsieve==2.1 stem==1.8.0 From 0a6d2bed2eaf5678afe4255bee0411bd8feb1159 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 16:16:04 +0300 Subject: [PATCH 28/72] Bump flake8 from 3.8.4 to 4.0.1 (#219) Bumps [flake8](https://github.com/pycqa/flake8) from 3.8.4 to 4.0.1. - [Release notes](https://github.com/pycqa/flake8/releases) - [Commits](https://github.com/pycqa/flake8/compare/3.8.4...4.0.1) --- updated-dependencies: - dependency-name: flake8 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6de052aa6..55036d41e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -flake8==3.8.4 +flake8==4.0.1 pytest==6.2.4 pytest-asyncio==0.14.0 pytest-cov==2.10.1 From 0bc11da598f1c3219d0364a68a8277e4448e5472 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 16:31:28 +0300 Subject: [PATCH 29/72] Bump aiohttp from 3.7.4 to 3.8.0 (#220) * Bump aiohttp from 3.7.4 to 3.8.0 Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.7.4 to 3.8.0. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.7.4...v3.8.0) --- updated-dependencies: - dependency-name: aiohttp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update requirements.txt Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: soxoj <31013580+soxoj@users.noreply.github.com> --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3f30d460a..506c12dd4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ aiodns==3.0.0 -aiohttp==3.7.4 +aiohttp==3.8.0 aiohttp-socks==0.5.5 arabic-reshaper==2.1.1 -async-timeout==3.0.1 +async-timeout==4.0.0 attrs==21.2.0 certifi==2020.12.5 chardet==3.0.4 From 3e370ce9675d564ca5c60dea9650073adca9b9f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 19:40:58 +0300 Subject: [PATCH 30/72] Bump aiohttp-socks from 0.5.5 to 0.6.0 (#222) * Bump aiohttp-socks from 0.5.5 to 0.6.0 Bumps [aiohttp-socks](https://github.com/romis2012/aiohttp-socks) from 0.5.5 to 0.6.0. - [Release notes](https://github.com/romis2012/aiohttp-socks/releases) - [Commits](https://github.com/romis2012/aiohttp-socks/compare/v0.5.5...v0.6.0) --- updated-dependencies: - dependency-name: aiohttp-socks dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update requirements.txt Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: soxoj <31013580+soxoj@users.noreply.github.com> --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 506c12dd4..d9ad97d54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ aiodns==3.0.0 aiohttp==3.8.0 -aiohttp-socks==0.5.5 +aiohttp-socks==0.6.0 arabic-reshaper==2.1.1 async-timeout==4.0.0 attrs==21.2.0 @@ -20,7 +20,6 @@ pycountry==20.7.3 PyPDF2==1.26.0 PySocks==1.7.1 python-bidi==0.4.2 -python-socks==1.1.2 requests==2.26.0 requests-futures==1.0.0 six==1.16.0 From dbcd52da811ce3ac9ef5aa39f9bfb873dede27e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 13:06:49 +0300 Subject: [PATCH 31/72] Bump typing-extensions from 3.7.4.3 to 3.10.0.2 (#224) Bumps [typing-extensions](https://github.com/python/typing) from 3.7.4.3 to 3.10.0.2. - [Release notes](https://github.com/python/typing/releases) - [Changelog](https://github.com/python/typing/blob/master/typing_extensions/CHANGELOG) - [Commits](https://github.com/python/typing/compare/3.7.4.3...3.10.0.2) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d9ad97d54..23e8044f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ soupsieve==2.1 stem==1.8.0 torrequest==0.1.0 tqdm==4.55.0 -typing-extensions==3.7.4.3 +typing-extensions==3.10.0.2 webencodings==0.5.1 xhtml2pdf==0.2.5 XMind==1.2.0 From 1283ac01bff7e91ac4a0022073116bc97f4739e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 13:07:00 +0300 Subject: [PATCH 32/72] Bump multidict from 5.1.0 to 5.2.0 (#225) Bumps [multidict](https://github.com/aio-libs/multidict) from 5.1.0 to 5.2.0. - [Release notes](https://github.com/aio-libs/multidict/releases) - [Changelog](https://github.com/aio-libs/multidict/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/multidict/compare/v5.1.0...v5.2.0) --- updated-dependencies: - dependency-name: multidict dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 23e8044f2..2e3757571 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ Jinja2==3.0.2 lxml==4.6.3 MarkupSafe==2.0.1 mock==4.0.2 -multidict==5.1.0 +multidict==5.2.0 pycountry==20.7.3 PyPDF2==1.26.0 PySocks==1.7.1 From 9e6d38dfead4edde4028a66f914a3f28b894ce24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 13:07:23 +0300 Subject: [PATCH 33/72] Bump idna from 2.10 to 3.3 (#228) Bumps [idna](https://github.com/kjd/idna) from 2.10 to 3.3. - [Release notes](https://github.com/kjd/idna/releases) - [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst) - [Commits](https://github.com/kjd/idna/compare/v2.10...v3.3) --- updated-dependencies: - dependency-name: idna dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2e3757571..100bbf6da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ colorama==0.4.4 future==0.18.2 future-annotations==1.0.0 html5lib==1.1 -idna==2.10 +idna==3.3 Jinja2==3.0.2 lxml==4.6.3 MarkupSafe==2.0.1 From 220c749af34d6ab0cf27fa37435215c096127129 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 13:07:41 +0300 Subject: [PATCH 34/72] Bump pytest-cov from 2.10.1 to 3.0.0 (#227) Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.10.1 to 3.0.0. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.10.1...v3.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 55036d41e..fb7c57fda 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ flake8==4.0.1 pytest==6.2.4 pytest-asyncio==0.14.0 -pytest-cov==2.10.1 +pytest-cov==3.0.0 pytest-httpserver==1.0.0 pytest-rerunfailures==9.1.1 From 447cd8511c26c79782992a97def8e15802dbddfe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 23:13:06 +0300 Subject: [PATCH 35/72] Bump mock from 4.0.2 to 4.0.3 (#226) Bumps [mock](https://github.com/testing-cabal/mock) from 4.0.2 to 4.0.3. - [Release notes](https://github.com/testing-cabal/mock/releases) - [Changelog](https://github.com/testing-cabal/mock/blob/master/CHANGELOG.rst) - [Commits](https://github.com/testing-cabal/mock/compare/4.0.2...4.0.3) --- updated-dependencies: - dependency-name: mock dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 100bbf6da..48d2a1e81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ idna==3.3 Jinja2==3.0.2 lxml==4.6.3 MarkupSafe==2.0.1 -mock==4.0.2 +mock==4.0.3 multidict==5.2.0 pycountry==20.7.3 PyPDF2==1.26.0 From b7b902f108b2e646e44c57b4e2cad788757c9c8e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 13:29:06 +0300 Subject: [PATCH 36/72] Bump certifi from 2020.12.5 to 2021.10.8 (#233) Bumps [certifi](https://github.com/certifi/python-certifi) from 2020.12.5 to 2021.10.8. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2020.12.05...2021.10.08) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 48d2a1e81..f7fe65333 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ aiohttp-socks==0.6.0 arabic-reshaper==2.1.1 async-timeout==4.0.0 attrs==21.2.0 -certifi==2020.12.5 +certifi==2021.10.8 chardet==3.0.4 colorama==0.4.4 future==0.18.2 From e4f87e1a9b084fb71809ade12a91e5bbd8c23142 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 13:29:14 +0300 Subject: [PATCH 37/72] Bump pytest-httpserver from 1.0.0 to 1.0.2 (#232) Bumps [pytest-httpserver](https://github.com/csernazs/pytest-httpserver) from 1.0.0 to 1.0.2. - [Release notes](https://github.com/csernazs/pytest-httpserver/releases) - [Changelog](https://github.com/csernazs/pytest-httpserver/blob/master/CHANGES.rst) - [Commits](https://github.com/csernazs/pytest-httpserver/compare/1.0.0...1.0.2) --- updated-dependencies: - dependency-name: pytest-httpserver dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index fb7c57fda..ddc92591c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,5 +2,5 @@ flake8==4.0.1 pytest==6.2.4 pytest-asyncio==0.14.0 pytest-cov==3.0.0 -pytest-httpserver==1.0.0 +pytest-httpserver==1.0.2 pytest-rerunfailures==9.1.1 From c5ac36affefce2ae5d90b64f07946b671dd7b527 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 13:29:23 +0300 Subject: [PATCH 38/72] Bump lxml from 4.6.3 to 4.6.4 (#231) Bumps [lxml](https://github.com/lxml/lxml) from 4.6.3 to 4.6.4. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.3...lxml-4.6.4) --- updated-dependencies: - dependency-name: lxml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f7fe65333..07521ed79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ future-annotations==1.0.0 html5lib==1.1 idna==3.3 Jinja2==3.0.2 -lxml==4.6.3 +lxml==4.6.4 MarkupSafe==2.0.1 mock==4.0.3 multidict==5.2.0 From b33656c02c10b1b1ec14b32cd9c237f9a2a3e382 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 13:30:00 +0300 Subject: [PATCH 39/72] Bump pefile from 2019.4.18 to 2021.9.3 (#229) Bumps [pefile](https://github.com/erocarrera/pefile) from 2019.4.18 to 2021.9.3. - [Release notes](https://github.com/erocarrera/pefile/releases) - [Commits](https://github.com/erocarrera/pefile/compare/v2019.4.18...v2021.9.3) --- updated-dependencies: - dependency-name: pefile dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyinstaller/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyinstaller/requirements.txt b/pyinstaller/requirements.txt index f03eae09e..7d6731dc8 100644 --- a/pyinstaller/requirements.txt +++ b/pyinstaller/requirements.txt @@ -1,5 +1,5 @@ maigret @ https://github.com/soxoj/maigret/archive/refs/heads/main.zip -pefile==2019.4.18 +pefile==2021.9.3 psutil==5.7.0 pyinstaller @ https://github.com/pyinstaller/pyinstaller/archive/develop.zip pywin32-ctypes==0.2.0 \ No newline at end of file From a7a56839a9051f8748b84bc557546b7006015c63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 13:42:15 +0300 Subject: [PATCH 40/72] Bump pytest-rerunfailures from 9.1.1 to 10.2 (#230) Bumps [pytest-rerunfailures](https://github.com/pytest-dev/pytest-rerunfailures) from 9.1.1 to 10.2. - [Release notes](https://github.com/pytest-dev/pytest-rerunfailures/releases) - [Changelog](https://github.com/pytest-dev/pytest-rerunfailures/blob/master/CHANGES.rst) - [Commits](https://github.com/pytest-dev/pytest-rerunfailures/compare/9.1.1...10.2) --- updated-dependencies: - dependency-name: pytest-rerunfailures dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ddc92591c..21cb5682d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,4 +3,4 @@ pytest==6.2.4 pytest-asyncio==0.14.0 pytest-cov==3.0.0 pytest-httpserver==1.0.2 -pytest-rerunfailures==9.1.1 +pytest-rerunfailures==10.2 From 6477a36ae1c13c75c7965ade81ec0f4685bac487 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 17:25:20 +0300 Subject: [PATCH 41/72] Bump yarl from 1.6.3 to 1.7.2 (#237) Bumps [yarl](https://github.com/aio-libs/yarl) from 1.6.3 to 1.7.2. - [Release notes](https://github.com/aio-libs/yarl/releases) - [Changelog](https://github.com/aio-libs/yarl/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/yarl/compare/v1.6.3...v1.7.2) --- updated-dependencies: - dependency-name: yarl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 07521ed79..2ef32b2c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,6 +32,6 @@ typing-extensions==3.10.0.2 webencodings==0.5.1 xhtml2pdf==0.2.5 XMind==1.2.0 -yarl==1.6.3 +yarl==1.7.2 networkx==2.5.1 pyvis==0.1.9 From 8b20799a34c1da130ebd387287061fca884c9386 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 17:26:02 +0300 Subject: [PATCH 42/72] Bump async-timeout from 4.0.0 to 4.0.1 (#236) Bumps [async-timeout](https://github.com/aio-libs/async-timeout) from 4.0.0 to 4.0.1. - [Release notes](https://github.com/aio-libs/async-timeout/releases) - [Changelog](https://github.com/aio-libs/async-timeout/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/async-timeout/compare/v4.0.0...v4.0.1) --- updated-dependencies: - dependency-name: async-timeout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2ef32b2c0..e6ef139fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ aiodns==3.0.0 aiohttp==3.8.0 aiohttp-socks==0.6.0 arabic-reshaper==2.1.1 -async-timeout==4.0.0 +async-timeout==4.0.1 attrs==21.2.0 certifi==2021.10.8 chardet==3.0.4 From 7f151a0d6a9c060153251d049fa22da3f0f54da3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 17:26:17 +0300 Subject: [PATCH 43/72] Bump psutil from 5.7.0 to 5.8.0 (#234) Bumps [psutil](https://github.com/giampaolo/psutil) from 5.7.0 to 5.8.0. - [Release notes](https://github.com/giampaolo/psutil/releases) - [Changelog](https://github.com/giampaolo/psutil/blob/master/HISTORY.rst) - [Commits](https://github.com/giampaolo/psutil/compare/release-5.7.0...release-5.8.0) --- updated-dependencies: - dependency-name: psutil dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyinstaller/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyinstaller/requirements.txt b/pyinstaller/requirements.txt index 7d6731dc8..1fb924a16 100644 --- a/pyinstaller/requirements.txt +++ b/pyinstaller/requirements.txt @@ -1,5 +1,5 @@ maigret @ https://github.com/soxoj/maigret/archive/refs/heads/main.zip pefile==2021.9.3 -psutil==5.7.0 +psutil==5.8.0 pyinstaller @ https://github.com/pyinstaller/pyinstaller/archive/develop.zip pywin32-ctypes==0.2.0 \ No newline at end of file From fa064b6c1ba0fb21cfa9aa7edacaa0e8d3c276d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 17:26:29 +0300 Subject: [PATCH 44/72] Bump jinja2 from 3.0.2 to 3.0.3 (#235) Bumps [jinja2](https://github.com/pallets/jinja) from 3.0.2 to 3.0.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.0.2...3.0.3) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e6ef139fd..02986e055 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ future==0.18.2 future-annotations==1.0.0 html5lib==1.1 idna==3.3 -Jinja2==3.0.2 +Jinja2==3.0.3 lxml==4.6.4 MarkupSafe==2.0.1 mock==4.0.3 From 0aec913eee0b15bd16c4d926dae411cb64b4fe6a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 17:26:41 +0300 Subject: [PATCH 45/72] Bump pytest from 6.2.4 to 6.2.5 (#238) Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.2.4 to 6.2.5. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.2.4...6.2.5) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 21cb5682d..a7ef2fe68 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ flake8==4.0.1 -pytest==6.2.4 +pytest==6.2.5 pytest-asyncio==0.14.0 pytest-cov==3.0.0 pytest-httpserver==1.0.2 From ef8f9ce15b659e89d7bb07121917c3a11d3e2237 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 17:42:11 +0300 Subject: [PATCH 46/72] Bump tqdm from 4.55.0 to 4.62.3 (#242) Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.55.0 to 4.62.3. - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.55.0...v4.62.3) --- updated-dependencies: - dependency-name: tqdm dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 02986e055..4cb23ee1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ socid-extractor>=0.0.21 soupsieve==2.1 stem==1.8.0 torrequest==0.1.0 -tqdm==4.55.0 +tqdm==4.62.3 typing-extensions==3.10.0.2 webencodings==0.5.1 xhtml2pdf==0.2.5 From fb75e9e5a21fb09dd0fceba3249e01e0d70c220c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 17:42:30 +0300 Subject: [PATCH 47/72] Bump arabic-reshaper from 2.1.1 to 2.1.3 (#243) Bumps [arabic-reshaper](https://github.com/mpcabd/python-arabic-reshaper) from 2.1.1 to 2.1.3. - [Release notes](https://github.com/mpcabd/python-arabic-reshaper/releases) - [Commits](https://github.com/mpcabd/python-arabic-reshaper/compare/v2.1.1...v2.1.3) --- updated-dependencies: - dependency-name: arabic-reshaper dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4cb23ee1b..9b576051c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiodns==3.0.0 aiohttp==3.8.0 aiohttp-socks==0.6.0 -arabic-reshaper==2.1.1 +arabic-reshaper==2.1.3 async-timeout==4.0.1 attrs==21.2.0 certifi==2021.10.8 From 954465f2d6dbd0b01fd9a305fd27a3ab962108c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 17:42:40 +0300 Subject: [PATCH 48/72] Bump pytest-asyncio from 0.14.0 to 0.16.0 (#240) Bumps [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) from 0.14.0 to 0.16.0. - [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases) - [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.14.0...v0.16.0) --- updated-dependencies: - dependency-name: pytest-asyncio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a7ef2fe68..9a0bafa64 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ flake8==4.0.1 pytest==6.2.5 -pytest-asyncio==0.14.0 +pytest-asyncio==0.16.0 pytest-cov==3.0.0 pytest-httpserver==1.0.2 pytest-rerunfailures==10.2 From d1c708e8c38ebefb17000912c9393476596ce408 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 17:43:26 +0300 Subject: [PATCH 49/72] Bump chardet from 3.0.4 to 4.0.0 (#241) Bumps [chardet](https://github.com/chardet/chardet) from 3.0.4 to 4.0.0. - [Release notes](https://github.com/chardet/chardet/releases) - [Commits](https://github.com/chardet/chardet/compare/3.0.4...4.0.0) --- updated-dependencies: - dependency-name: chardet dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9b576051c..efab37bd2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ arabic-reshaper==2.1.3 async-timeout==4.0.1 attrs==21.2.0 certifi==2021.10.8 -chardet==3.0.4 +chardet==4.0.0 colorama==0.4.4 future==0.18.2 future-annotations==1.0.0 From 39a523c1887061f8a6f3e89b92021c8e9101b943 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 17:43:38 +0300 Subject: [PATCH 50/72] Bump soupsieve from 2.1 to 2.3.1 (#239) Bumps [soupsieve](https://github.com/facelessuser/soupsieve) from 2.1 to 2.3.1. - [Release notes](https://github.com/facelessuser/soupsieve/releases) - [Commits](https://github.com/facelessuser/soupsieve/compare/2.1.0...2.3.1) --- updated-dependencies: - dependency-name: soupsieve dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index efab37bd2..dc2b36126 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ requests==2.26.0 requests-futures==1.0.0 six==1.16.0 socid-extractor>=0.0.21 -soupsieve==2.1 +soupsieve==2.3.1 stem==1.8.0 torrequest==0.1.0 tqdm==4.62.3 From b664efc3f11744bf474fad10e874a0b82a447818 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Nov 2021 22:56:34 +0300 Subject: [PATCH 51/72] Bump aiohttp from 3.8.0 to 3.8.1 (#246) Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.8.0 to 3.8.1. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/v3.8.1/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.8.0...v3.8.1) --- updated-dependencies: - dependency-name: aiohttp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dc2b36126..a8caba24f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ aiodns==3.0.0 -aiohttp==3.8.0 +aiohttp==3.8.1 aiohttp-socks==0.6.0 arabic-reshaper==2.1.3 async-timeout==4.0.1 From 09ae96e4da2e039f18cf9c48d4bb4b119109d96c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Nov 2021 22:56:48 +0300 Subject: [PATCH 52/72] Bump typing-extensions from 3.10.0.2 to 4.0.0 (#245) Bumps [typing-extensions](https://github.com/python/typing) from 3.10.0.2 to 4.0.0. - [Release notes](https://github.com/python/typing/releases) - [Changelog](https://github.com/python/typing/blob/master/typing_extensions/CHANGELOG) - [Commits](https://github.com/python/typing/compare/3.10.0.2...4.0.0) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a8caba24f..0eb34c679 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ soupsieve==2.3.1 stem==1.8.0 torrequest==0.1.0 tqdm==4.62.3 -typing-extensions==3.10.0.2 +typing-extensions==4.0.0 webencodings==0.5.1 xhtml2pdf==0.2.5 XMind==1.2.0 From 90ec62d6576f6c0d926c326677489557ed756ba4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Nov 2021 00:22:00 +0300 Subject: [PATCH 53/72] Bump aiohttp-socks from 0.6.0 to 0.6.1 (#249) Bumps [aiohttp-socks](https://github.com/romis2012/aiohttp-socks) from 0.6.0 to 0.6.1. - [Release notes](https://github.com/romis2012/aiohttp-socks/releases) - [Commits](https://github.com/romis2012/aiohttp-socks/compare/v0.6.0...v0.6.1) --- updated-dependencies: - dependency-name: aiohttp-socks dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0eb34c679..26dc46a24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ aiodns==3.0.0 aiohttp==3.8.1 -aiohttp-socks==0.6.0 +aiohttp-socks==0.6.1 arabic-reshaper==2.1.3 async-timeout==4.0.1 attrs==21.2.0 From c154cf9f237b98cc63d80f927194ca722e283de0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Nov 2021 14:22:33 +0300 Subject: [PATCH 54/72] Bump aiohttp-socks from 0.6.1 to 0.7.1 (#250) Bumps [aiohttp-socks](https://github.com/romis2012/aiohttp-socks) from 0.6.1 to 0.7.1. - [Release notes](https://github.com/romis2012/aiohttp-socks/releases) - [Commits](https://github.com/romis2012/aiohttp-socks/compare/v0.6.1...v0.7.1) --- updated-dependencies: - dependency-name: aiohttp-socks dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 26dc46a24..f8546aa0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ aiodns==3.0.0 aiohttp==3.8.1 -aiohttp-socks==0.6.1 +aiohttp-socks==0.7.1 arabic-reshaper==2.1.3 async-timeout==4.0.1 attrs==21.2.0 From 2b905ae996de940499d20f708d02ddcf1f24fc65 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Dec 2021 18:01:42 +0300 Subject: [PATCH 55/72] Bump typing-extensions from 4.0.0 to 4.0.1 (#253) Bumps [typing-extensions](https://github.com/python/typing) from 4.0.0 to 4.0.1. - [Release notes](https://github.com/python/typing/releases) - [Changelog](https://github.com/python/typing/blob/master/typing_extensions/CHANGELOG) - [Commits](https://github.com/python/typing/commits) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f8546aa0f..47c52918b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ soupsieve==2.3.1 stem==1.8.0 torrequest==0.1.0 tqdm==4.62.3 -typing-extensions==4.0.0 +typing-extensions==4.0.1 webencodings==0.5.1 xhtml2pdf==0.2.5 XMind==1.2.0 From f533c30564c78a8e777287e553ddb61cc20f20ee Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Wed, 8 Dec 2021 04:21:16 +0300 Subject: [PATCH 56/72] Fixed some false positives (#254) --- maigret/resources/data.json | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 3de56a764..e14a1a79d 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -6465,10 +6465,16 @@ }, "Hackerearth": { "tags": [ - "in" + "freelance" ], - "checkType": "status_code", + "checkType": "message", "alexaRank": 7807, + "absenceStrs": [ + "404. URL not found. Date: Thu, 9 Dec 2021 00:50:53 +0300 Subject: [PATCH 57/72] Disabled non-working sites (#255) * Disabled non-working sites * Added invalid result issue template --- .github/ISSUE_TEMPLATE/report-false-result.md | 10 ++++++++++ maigret/resources/data.json | 11 +++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/report-false-result.md diff --git a/.github/ISSUE_TEMPLATE/report-false-result.md b/.github/ISSUE_TEMPLATE/report-false-result.md new file mode 100644 index 000000000..e65216fab --- /dev/null +++ b/.github/ISSUE_TEMPLATE/report-false-result.md @@ -0,0 +1,10 @@ +--- +name: Report invalid result +about: I want report invalid result of Maigret Search +title: Invalit result +labels: false-result +assignees: soxoj + +--- + +Invalid link: diff --git a/maigret/resources/data.json b/maigret/resources/data.json index e14a1a79d..cecc0145e 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -3901,6 +3901,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Duno": { + "disabled": true, "tags": [ "in", "pk" @@ -9450,6 +9451,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Mylove": { + "disabled": true, "tags": [ "ru" ], @@ -10603,6 +10605,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Pilguy": { + "disabled": true, "tags": [ "forum", "ru" @@ -10737,6 +10740,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Playlists": { + "disabled": true, "tags": [ "in", "us" @@ -10777,11 +10781,9 @@ "usernameUnclaimed": "noonewouldeverusethis" }, "Plug.DJ": { + "disabled": true, "tags": [ - "fr", - "gb", - "kr", - "us" + "music" ], "checkType": "status_code", "alexaRank": 3799094, @@ -17241,6 +17243,7 @@ ] }, "forum.cxem.net": { + "disabled": true, "tags": [ "forum", "ru" From 4880b7124672e9beb3dc48d68a52cd71e809d3a5 Mon Sep 17 00:00:00 2001 From: soxoj <31013580+soxoj@users.noreply.github.com> Date: Thu, 9 Dec 2021 01:04:24 +0300 Subject: [PATCH 58/72] Added false results buttons to reports, fixed some falses (#256) --- .github/ISSUE_TEMPLATE/report-false-result.md | 4 ++-- maigret/resources/data.json | 5 ++++- maigret/resources/simple_report.tpl | 1 + maigret/resources/simple_report_pdf.css | 4 ++++ maigret/resources/simple_report_pdf.tpl | 1 + 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/report-false-result.md b/.github/ISSUE_TEMPLATE/report-false-result.md index e65216fab..fae9cd7e0 100644 --- a/.github/ISSUE_TEMPLATE/report-false-result.md +++ b/.github/ISSUE_TEMPLATE/report-false-result.md @@ -1,7 +1,7 @@ --- name: Report invalid result -about: I want report invalid result of Maigret Search -title: Invalit result +about: I want to report invalid result of Maigret search +title: Invalid result labels: false-result assignees: soxoj diff --git a/maigret/resources/data.json b/maigret/resources/data.json index cecc0145e..1cbe8033b 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -5383,6 +5383,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Funnyordie": { + "disabled": true, "tags": [ "in", "us" @@ -10149,7 +10150,7 @@ ], "checkType": "message", "absenceStrs": [ - "404" + "404 - Not Found" ], "alexaRank": 25200, "urlMain": "https://overclockers.ru", @@ -10426,6 +10427,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "PeopleAndCountries": { + "disabled": true, "tags": [ "forum", "ru" @@ -17524,6 +17526,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "funny-games.biz": { + "disabled": true, "tags": [ "forum", "lt", diff --git a/maigret/resources/simple_report.tpl b/maigret/resources/simple_report.tpl index c2e3322c8..7c3c48efa 100644 --- a/maigret/resources/simple_report.tpl +++ b/maigret/resources/simple_report.tpl @@ -68,6 +68,7 @@
+ Invalid? Photo

diff --git a/maigret/resources/simple_report_pdf.css b/maigret/resources/simple_report_pdf.css index 9fda1b407..e2282d72a 100644 --- a/maigret/resources/simple_report_pdf.css +++ b/maigret/resources/simple_report_pdf.css @@ -38,4 +38,8 @@ div { border-bottom-color: #3e3e3e; border-bottom-width: 1px; border-bottom-style: solid; +} +.invalid-button { + position: absolute; + left: 10px; } \ No newline at end of file diff --git a/maigret/resources/simple_report_pdf.tpl b/maigret/resources/simple_report_pdf.tpl index f3db39584..6a532aa40 100644 --- a/maigret/resources/simple_report_pdf.tpl +++ b/maigret/resources/simple_report_pdf.tpl @@ -64,6 +64,7 @@
+ Invalid?
From 262f97ce330844159bc4264b2400ec7a4c862747 Mon Sep 17 00:00:00 2001 From: Soxoj <31013580+soxoj@users.noreply.github.com> Date: Fri, 10 Dec 2021 23:01:56 +0300 Subject: [PATCH 59/72] Fixed xHamster, added support of proxies to self-check mode (#259) --- maigret/checking.py | 5 ++++- maigret/maigret.py | 1 + maigret/resources/data.json | 8 +++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/maigret/checking.py b/maigret/checking.py index 603eeee39..9f368f74e 100644 --- a/maigret/checking.py +++ b/maigret/checking.py @@ -764,6 +764,7 @@ async def site_self_check( semaphore, db: MaigretDatabase, silent=False, + proxy=None, tor_proxy=None, i2p_proxy=None, ): @@ -789,6 +790,7 @@ async def site_self_check( forced=True, no_progressbar=True, retries=1, + proxy=proxy, tor_proxy=tor_proxy, i2p_proxy=i2p_proxy, ) @@ -845,6 +847,7 @@ async def self_check( logger, silent=False, max_connections=10, + proxy=None, tor_proxy=None, i2p_proxy=None, ) -> bool: @@ -859,7 +862,7 @@ def disabled_count(lst): for _, site in all_sites.items(): check_coro = site_self_check( - site, logger, sem, db, silent, tor_proxy, i2p_proxy + site, logger, sem, db, silent, proxy, tor_proxy, i2p_proxy ) future = asyncio.ensure_future(check_coro) tasks.append(future) diff --git a/maigret/maigret.py b/maigret/maigret.py index 3a5dac011..a72274087 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -548,6 +548,7 @@ async def main(): db, site_data, logger, + proxy=args.proxy, max_connections=args.connections, tor_proxy=args.tor_proxy, i2p_proxy=args.i2p_proxy, diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 1cbe8033b..a259797df 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -19922,7 +19922,13 @@ "porn", "us" ], - "checkType": "status_code", + "checkType": "message", + "presenseStrs": [ + "user-info-section" + ], + "absenceStrs": [ + "User not found" + ], "alexaRank": 136, "urlMain": "https://xhamster.com", "url": "https://xhamster.com/users/{username}", From aee9125c964d926a95f81f543bdda4089f61a3e8 Mon Sep 17 00:00:00 2001 From: Soxoj <31013580+soxoj@users.noreply.github.com> Date: Sun, 12 Dec 2021 14:00:11 +0300 Subject: [PATCH 60/72] Disabled non-working sites, updated public sites list (#263) --- maigret/resources/data.json | 12 ++++-- sites.md | 78 ++++++++++++++++++++++--------------- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/maigret/resources/data.json b/maigret/resources/data.json index a259797df..82c5c889c 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -1720,6 +1720,7 @@ }, "Bayoushooter": { "urlSubpath": "/forums", + "disabled": true, "tags": [ "forum", "pk", @@ -2402,6 +2403,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "CapFriendly": { + "disabled": true, "tags": [ "ca", "us" @@ -4499,6 +4501,7 @@ }, "Fapforum": { "urlSubpath": "/forum", + "disabled": true, "engine": "vBulletin", "alexaRank": 954536, "urlMain": "http://fapforum.net", @@ -12150,6 +12153,7 @@ "ru", "ua" ], + "disabled": true, "engine": "vBulletin", "alexaRank": 491084, "urlMain": "https://www.rusforum.com/", @@ -14883,8 +14887,8 @@ }, "Vgtimes/Games": { "tags": [ - "ru", - "forum" + "forum", + "ru" ], "checkType": "status_code", "alexaRank": 17926, @@ -28346,9 +28350,9 @@ "checkType": "message", "alexaRank": 50263, "tags": [ + "by", "news", - "tech", - "by" + "tech" ] }, "Vgtimes": { diff --git a/sites.md b/sites.md index 7bc912e9c..859747a6f 100644 --- a/sites.md +++ b/sites.md @@ -1,9 +1,10 @@ -## List of supported sites (search methods): total 2560 +## List of supported sites (search methods): total 2574 Rank data fetched from Alexa by domains. -1. ![](https://www.google.com/s2/favicons?domain=https://maps.google.com/) [GoogleMaps (https://maps.google.com/)](https://maps.google.com/)*: top 1, maps, us* +1. ![](https://www.google.com/s2/favicons?domain=https://maps.google.com/) [Google Maps (https://maps.google.com/)](https://maps.google.com/)*: top 1, maps, us* +1. ![](https://www.google.com/s2/favicons?domain=https://plus.google.com) [Google Plus (archived) (https://plus.google.com)](https://plus.google.com)*: top 1* 1. ![](https://www.google.com/s2/favicons?domain=https://play.google.com/store) [GooglePlayStore (https://play.google.com/store)](https://play.google.com/store)*: top 1, apps, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.youtube.com/) [YouTube (https://www.youtube.com/)](https://www.youtube.com/)*: top 2, us, video* 1. ![](https://www.google.com/s2/favicons?domain=https://www.facebook.com/) [Facebook (https://www.facebook.com/)](https://www.facebook.com/)*: top 10, networking* @@ -316,6 +317,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://club.doctissimo.fr) [doctissimo (https://club.doctissimo.fr)](https://club.doctissimo.fr)*: top 10K, fr* 1. ![](https://www.google.com/s2/favicons?domain=https://www.anime-planet.com) [Anime-planet (https://www.anime-planet.com)](https://www.anime-planet.com)*: top 10K, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.dailykos.com) [dailykos (https://www.dailykos.com)](https://www.dailykos.com)*: top 10K, us* +1. ![](https://www.google.com/s2/favicons?domain=https://blogs.klerk.ru) [blogs.klerk.ru (https://blogs.klerk.ru)](https://blogs.klerk.ru)*: top 10K* 1. ![](https://www.google.com/s2/favicons?domain=https://pokemonshowdown.com) [Pokemon Showdown (https://pokemonshowdown.com)](https://pokemonshowdown.com)*: top 10K, us* 1. ![](https://www.google.com/s2/favicons?domain=https://coub.com/) [Coub (https://coub.com/)](https://coub.com/)*: top 10K, us* 1. ![](https://www.google.com/s2/favicons?domain=https://forum.corsair.com) [CORSAIR (https://forum.corsair.com)](https://forum.corsair.com)*: top 10K, forum, us* @@ -332,7 +334,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.trakt.tv/) [Trakt (https://www.trakt.tv/)](https://www.trakt.tv/)*: top 10K, de, fr* 1. ![](https://www.google.com/s2/favicons?domain=https://hotcopper.com.au) [Hotcopper (https://hotcopper.com.au)](https://hotcopper.com.au)*: top 10K, au* 1. ![](https://www.google.com/s2/favicons?domain=https://pandia.ru) [Pandia (https://pandia.ru)](https://pandia.ru)*: top 10K, news, ru* -1. ![](https://www.google.com/s2/favicons?domain=https://www.hackerearth.com) [Hackerearth (https://www.hackerearth.com)](https://www.hackerearth.com)*: top 10K, in* +1. ![](https://www.google.com/s2/favicons?domain=https://www.hackerearth.com) [Hackerearth (https://www.hackerearth.com)](https://www.hackerearth.com)*: top 10K, freelance* 1. ![](https://www.google.com/s2/favicons?domain=https://repl.it/) [Repl.it (https://repl.it/)](https://repl.it/)*: top 10K, coding, us* 1. ![](https://www.google.com/s2/favicons?domain=https://support.advancedcustomfields.com/) [AdvancedCustomFields (https://support.advancedcustomfields.com/)](https://support.advancedcustomfields.com/)*: top 10K, au, in, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.seoclerks.com) [SeoClerks (https://www.seoclerks.com)](https://www.seoclerks.com)*: top 10K, in, jp, us* @@ -428,11 +430,12 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.theodysseyonline.com) [TheOdysseyOnline (https://www.theodysseyonline.com)](https://www.theodysseyonline.com)*: top 100K, blog* 1. ![](https://www.google.com/s2/favicons?domain=https://dtf.ru) [DTF (https://dtf.ru)](https://dtf.ru)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://trashbox.ru/) [TRASHBOX.RU (https://trashbox.ru/)](https://trashbox.ru/)*: top 100K, az, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://vgtimes.ru) [Vgtimes (https://vgtimes.ru)](https://vgtimes.ru)*: top 100K, gaming, news, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://launchpad.net/) [Launchpad (https://launchpad.net/)](https://launchpad.net/)*: top 100K, tech, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.digitalpoint.com) [DigitalPoint (https://www.digitalpoint.com)](https://www.digitalpoint.com)*: top 100K, forum* 1. ![](https://www.google.com/s2/favicons?domain=https://forums.digitalpoint.com/) [forums.digitalpoint.com (https://forums.digitalpoint.com/)](https://forums.digitalpoint.com/)*: top 100K, forum, in* 1. ![](https://www.google.com/s2/favicons?domain=https://www.voices.com/) [Voices (https://www.voices.com/)](https://www.voices.com/)*: top 100K, us* -1. ![](https://www.google.com/s2/favicons?domain=https://forums.overclockers.co.uk) [forums.overclockers.co.uk (https://forums.overclockers.co.uk)](https://forums.overclockers.co.uk)*: top 100K, forum, gb, uk* +1. ![](https://www.google.com/s2/favicons?domain=https://forums.overclockers.co.uk) [forums.overclockers.co.uk (https://forums.overclockers.co.uk)](https://forums.overclockers.co.uk)*: top 100K, forum, gb, uk*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=http://forum.lingvolive.com) [Lingvolive (http://forum.lingvolive.com)](http://forum.lingvolive.com)*: top 100K, de, forum, it, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.ariva.de/) [Ariva (https://www.ariva.de/)](https://www.ariva.de/)*: top 100K, de* 1. ![](https://www.google.com/s2/favicons?domain=https://talks.by) [Talks.by (https://talks.by)](https://talks.by)*: top 100K, by, forum*, search is disabled @@ -440,7 +443,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.techdirt.com/) [Techdirt (https://www.techdirt.com/)](https://www.techdirt.com/)*: top 100K, in, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.myminifactory.com/) [MyMiniFactory (https://www.myminifactory.com/)](https://www.myminifactory.com/)*: top 100K, gb, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.geocaching.com/) [geocaching (https://www.geocaching.com/)](https://www.geocaching.com/)*: top 100K, de, hobby, us* -1. ![](https://www.google.com/s2/favicons?domain=https://vgtimes.ru) [Vgtimes (https://vgtimes.ru)](https://vgtimes.ru)*: top 100K, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://vgtimes.ru) [Vgtimes/Games (https://vgtimes.ru)](https://vgtimes.ru)*: top 100K, forum, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://cont.ws) [Cont (https://cont.ws)](https://cont.ws)*: top 100K, be, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.etxt.ru) [Etxt (https://www.etxt.ru)](https://www.etxt.ru)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.babyblog.ru/) [BabyBlog.ru (https://www.babyblog.ru/)](https://www.babyblog.ru/)*: top 100K, ru* @@ -469,7 +472,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.codewars.com) [Codewars (https://www.codewars.com)](https://www.codewars.com)*: top 100K, coding, us* 1. ![](https://www.google.com/s2/favicons?domain=https://shikimori.one) [Shikimori (https://shikimori.one)](https://shikimori.one)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://gurushots.com/) [GuruShots (https://gurushots.com/)](https://gurushots.com/)*: top 100K, us* -1. ![](https://www.google.com/s2/favicons?domain=https://id.pr-cy.ru) [PRCY (https://id.pr-cy.ru)](https://id.pr-cy.ru)*: top 100K, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://id.pr-cy.ru) [PRCY (https://id.pr-cy.ru)](https://id.pr-cy.ru)*: top 100K, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=http://forum.igromania.ru/) [igromania (http://forum.igromania.ru/)](http://forum.igromania.ru/)*: top 100K, forum, gaming, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://facenama.com/) [Facenama (https://facenama.com/)](https://facenama.com/)*: top 100K, ir* 1. ![](https://www.google.com/s2/favicons?domain=http://www.pushsquare.com) [PushSquare (http://www.pushsquare.com)](http://www.pushsquare.com)*: top 100K, gaming, news, us* @@ -483,7 +486,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://robertsspaceindustries.com/) [Star Citizen (https://robertsspaceindustries.com/)](https://robertsspaceindustries.com/)*: top 100K, de, us* 1. ![](https://www.google.com/s2/favicons?domain=https://eva.ru/) [Eva (https://eva.ru/)](https://eva.ru/)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://e621.net) [E621 (https://e621.net)](https://e621.net)*: top 100K, us* -1. ![](https://www.google.com/s2/favicons?domain=https://www.metacafe.com/) [Metacafe (https://www.metacafe.com/)](https://www.metacafe.com/)*: top 100K, in, us* +1. ![](https://www.google.com/s2/favicons?domain=https://www.metacafe.com/) [Metacafe (https://www.metacafe.com/)](https://www.metacafe.com/)*: top 100K, in, us*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://discussions.tomtom.com/) [Tomtom (https://discussions.tomtom.com/)](https://discussions.tomtom.com/)*: top 100K, de, in, it, nl, no, us* 1. ![](https://www.google.com/s2/favicons?domain=https://tryhackme.com/) [TryHackMe (https://tryhackme.com/)](https://tryhackme.com/)*: top 100K, hacking, in, us* 1. ![](https://www.google.com/s2/favicons?domain=https://gbatemp.net/) [GBAtemp.net (https://gbatemp.net/)](https://gbatemp.net/)*: top 100K, de, forum, gaming, us* @@ -559,7 +562,8 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.goldderby.com) [Goldderby (https://www.goldderby.com)](https://www.goldderby.com)*: top 100K, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.blackhatprotools.info) [BlackHatProTools (https://www.blackhatprotools.info)](https://www.blackhatprotools.info)*: top 100K, forum* 1. ![](https://www.google.com/s2/favicons?domain=https://www.aufeminin.com) [Aufeminin (https://www.aufeminin.com)](https://www.aufeminin.com)*: top 100K, fr, ma, mg* -1. ![](https://www.google.com/s2/favicons?domain=https://www.vsemayki.ru/) [Vsemayki (https://www.vsemayki.ru/)](https://www.vsemayki.ru/)*: top 100K, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://www.vsemayki.ru/) [Vsemayki (https://www.vsemayki.ru/)](https://www.vsemayki.ru/)*: top 100K, ru*, search is disabled +1. ![](https://www.google.com/s2/favicons?domain=https://likee.video) [Likee (https://likee.video)](https://likee.video)*: top 100K, video* 1. ![](https://www.google.com/s2/favicons?domain=https://blog.7ya.ru) [7ya (https://blog.7ya.ru)](https://blog.7ya.ru)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://club.7ya.ru) [club.7ya.ru (https://club.7ya.ru)](https://club.7ya.ru)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://gloria.tv) [gloria.tv (https://gloria.tv)](https://gloria.tv)*: top 100K, ar, mx, pl, sk, us* @@ -600,7 +604,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=http://blogmarks.net) [Blogmarks (http://blogmarks.net)](http://blogmarks.net)*: top 100K, fr, in* 1. ![](https://www.google.com/s2/favicons?domain=http://forum.oszone.net) [ForumOszone (http://forum.oszone.net)](http://forum.oszone.net)*: top 100K, forum, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://davesgarden.com) [Davesgarden (https://davesgarden.com)](https://davesgarden.com)*: top 100K, us* -1. ![](https://www.google.com/s2/favicons?domain=https://forum.cxem.net/) [forum.cxem.net (https://forum.cxem.net/)](https://forum.cxem.net/)*: top 100K, forum, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://forum.cxem.net/) [forum.cxem.net (https://forum.cxem.net/)](https://forum.cxem.net/)*: top 100K, forum, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://icq.com) [ICQ (https://icq.com)](https://icq.com)*: top 100K, ch, ru, tr* 1. ![](https://www.google.com/s2/favicons?domain=https://d3.ru/) [d3 (https://d3.ru/)](https://d3.ru/)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://forum.dwg.ru/) [dwg (https://forum.dwg.ru/)](https://forum.dwg.ru/)*: top 100K, forum, ru* @@ -610,7 +614,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=http://promodj.com/) [PromoDJ (http://promodj.com/)](http://promodj.com/)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.sythe.org) [Sythe (https://www.sythe.org)](https://www.sythe.org)*: top 100K, ca, forum, us* 1. ![](https://www.google.com/s2/favicons?domain=http://www.forum.hr) [forum.hr (http://www.forum.hr)](http://www.forum.hr)*: top 100K, forum, hr* -1. ![](https://www.google.com/s2/favicons?domain=https://www.capfriendly.com/) [CapFriendly (https://www.capfriendly.com/)](https://www.capfriendly.com/)*: top 100K, ca, us* +1. ![](https://www.google.com/s2/favicons?domain=https://www.capfriendly.com/) [CapFriendly (https://www.capfriendly.com/)](https://www.capfriendly.com/)*: top 100K, ca, us*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://kosmetista.ru) [Kosmetista (https://kosmetista.ru)](https://kosmetista.ru)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=http://forum.notebookreview.com) [NotebookReview (http://forum.notebookreview.com)](http://forum.notebookreview.com)*: top 100K, forum, in, us* 1. ![](https://www.google.com/s2/favicons?domain=https://tl.net) [Tl (https://tl.net)](https://tl.net)*: top 100K, de, dk, us* @@ -621,7 +625,8 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.ethicalhacker.net) [EthicalHacker (https://www.ethicalhacker.net)](https://www.ethicalhacker.net)*: top 100K, in, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.playstationtrophies.org) [PlaystationTrophies (https://www.playstationtrophies.org)](https://www.playstationtrophies.org)*: top 100K, forum, gaming* 1. ![](https://www.google.com/s2/favicons?domain=https://www.hr.com) [Hr (https://www.hr.com)](https://www.hr.com)*: top 100K, in, us* -1. ![](https://www.google.com/s2/favicons?domain=https://www.funnyordie.com) [Funnyordie (https://www.funnyordie.com)](https://www.funnyordie.com)*: top 100K, in, us* +1. ![](https://www.google.com/s2/favicons?domain=https://www.funnyordie.com) [Funnyordie (https://www.funnyordie.com)](https://www.funnyordie.com)*: top 100K, in, us*, search is disabled +1. ![](https://www.google.com/s2/favicons?domain=https://id.dev.by) [Dev.by (https://id.dev.by)](https://id.dev.by)*: top 100K, by, news, tech* 1. ![](https://www.google.com/s2/favicons?domain=http://forum.hochu.ua) [hochu (http://forum.hochu.ua)](http://forum.hochu.ua)*: top 100K, forum, ru, ua* 1. ![](https://www.google.com/s2/favicons?domain=https://24open.ru) [24open (https://24open.ru)](https://24open.ru)*: top 100K, dating, ru, us*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.pokecommunity.com) [Pokecommunity (https://www.pokecommunity.com)](https://www.pokecommunity.com)*: top 100K, de, forum, gb, us* @@ -718,8 +723,8 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://steamidfinder.com) [Steamidfinder (https://steamidfinder.com)](https://steamidfinder.com)*: top 100K, gaming* 1. ![](https://www.google.com/s2/favicons?domain=https://steamidfinder.com) [Steamidfinder (by id) (https://steamidfinder.com)](https://steamidfinder.com)*: top 100K, gaming* 1. ![](https://www.google.com/s2/favicons?domain=https://mssg.me) [mssg.me (https://mssg.me)](https://mssg.me)*: top 100K, ru* -1. ![](https://www.google.com/s2/favicons?domain=http://capitalcity.combats.com) [CapitalcityCombats (http://capitalcity.combats.com)](http://capitalcity.combats.com)*: top 100K, az, it, ru* -1. ![](https://www.google.com/s2/favicons?domain=http://demonscity.combats.com) [Demonscity (http://demonscity.combats.com)](http://demonscity.combats.com)*: top 100K, az, it, pa* +1. ![](https://www.google.com/s2/favicons?domain=http://capitalcity.combats.com) [CapitalcityCombats (http://capitalcity.combats.com)](http://capitalcity.combats.com)*: top 100K, ru* +1. ![](https://www.google.com/s2/favicons?domain=http://demonscity.combats.com) [Demonscity (http://demonscity.combats.com)](http://demonscity.combats.com)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.cfd-online.com) [cfd-online (https://www.cfd-online.com)](https://www.cfd-online.com)*: top 100K, ca, de, fr, in, us* 1. ![](https://www.google.com/s2/favicons?domain=https://otzyvy.pro) [Otzyvy (https://otzyvy.pro)](https://otzyvy.pro)*: top 100K, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.allthelyrics.com) [AllTheLyrics (https://www.allthelyrics.com)](https://www.allthelyrics.com)*: top 100K, forum, music* @@ -746,10 +751,10 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://asciinema.org) [Asciinema (https://asciinema.org)](https://asciinema.org)*: top 10M, in, tr, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.ddo.com) [Ddo (https://www.ddo.com)](https://www.ddo.com)*: top 10M, forum, us* 1. ![](https://www.google.com/s2/favicons?domain=https://golbis.com) [Golbis (https://golbis.com)](https://golbis.com)*: top 10M, ru* -1. ![](https://www.google.com/s2/favicons?domain=https://forexdengi.com/) [ForexDengi (https://forexdengi.com/)](https://forexdengi.com/)*: top 10M, forum, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://forexdengi.com/) [ForexDengi (https://forexdengi.com/)](https://forexdengi.com/)*: top 10M, forum, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://mywed.com/ru) [Mywed (https://mywed.com/ru)](https://mywed.com/ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://elixirforum.com) [Elixirforum (https://elixirforum.com)](https://elixirforum.com)*: top 10M, coding, forum* -1. ![](https://www.google.com/s2/favicons?domain=https://forums.funny-games.biz) [funny-games.biz (https://forums.funny-games.biz)](https://forums.funny-games.biz)*: top 10M, forum, lt, us* +1. ![](https://www.google.com/s2/favicons?domain=https://forums.funny-games.biz) [funny-games.biz (https://forums.funny-games.biz)](https://forums.funny-games.biz)*: top 10M, forum, lt, us*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=http://wolpy.com) [Wolpy (http://wolpy.com)](http://wolpy.com)*: top 10M, travel* 1. ![](https://www.google.com/s2/favicons?domain=http://www.tfw2005.com/boards/) [tfw2005.com (http://www.tfw2005.com/boards/)](http://www.tfw2005.com/boards/)*: top 10M, forum, us* 1. ![](https://www.google.com/s2/favicons?domain=https://truesteamachievements.com) [Truesteamachievements (https://truesteamachievements.com)](https://truesteamachievements.com)*: top 10M, az, gb, us* @@ -850,7 +855,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.kharkovforum.com/) [KharkovForum (https://www.kharkovforum.com/)](https://www.kharkovforum.com/)*: top 10M, forum, ua* 1. ![](https://www.google.com/s2/favicons?domain=https://studwork.org/) [Studwork (https://studwork.org/)](https://studwork.org/)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://forum.freeton.org) [forum.freeton.org (https://forum.freeton.org)](https://forum.freeton.org)*: top 10M, finance, forum* -1. ![](https://www.google.com/s2/favicons?domain=https://playlists.net) [Playlists (https://playlists.net)](https://playlists.net)*: top 10M, in, us* +1. ![](https://www.google.com/s2/favicons?domain=https://playlists.net) [Playlists (https://playlists.net)](https://playlists.net)*: top 10M, in, us*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://liberapay.com) [Liberapay (https://liberapay.com)](https://liberapay.com)*: top 10M, eg, finance, in, pk, us, za* 1. ![](https://www.google.com/s2/favicons?domain=https://forum.artinvestment.ru/) [artinvestment (https://forum.artinvestment.ru/)](https://forum.artinvestment.ru/)*: top 10M, forum, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://forum.golangbridge.org/) [Golangbridge (https://forum.golangbridge.org/)](https://forum.golangbridge.org/)*: top 10M, forum, in, sa, ua, us, vn* @@ -917,6 +922,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=http://muz-fresh.ucoz.kz) [muz-fresh.ucoz.kz (http://muz-fresh.ucoz.kz)](http://muz-fresh.ucoz.kz)*: top 10M, kz* 1. ![](https://www.google.com/s2/favicons?domain=https://radiomed.ru) [Radiomed (https://radiomed.ru)](https://radiomed.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=http://excelworld.ru) [excelworld.ru (http://excelworld.ru)](http://excelworld.ru)*: top 10M, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://onlyfinder.com) [Onlyfinder (https://onlyfinder.com)](https://onlyfinder.com)*: top 10M, webcam* 1. ![](https://www.google.com/s2/favicons?domain=https://binarysearch.com/) [BinarySearch (https://binarysearch.com/)](https://binarysearch.com/)*: top 10M, in* 1. ![](https://www.google.com/s2/favicons?domain=https://nick-name.ru/) [Nick-name.ru (https://nick-name.ru/)](https://nick-name.ru/)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://naturalworld.guru) [Naturalworld (https://naturalworld.guru)](https://naturalworld.guru)*: top 10M, ru* @@ -995,7 +1001,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.meendo.net) [Meendo (https://www.meendo.net)](https://www.meendo.net)*: top 10M, bg, kg, ru, ua* 1. ![](https://www.google.com/s2/favicons?domain=https://www.oakleyforum.com) [oakleyforum.com (https://www.oakleyforum.com)](https://www.oakleyforum.com)*: top 10M, forum* 1. ![](https://www.google.com/s2/favicons?domain=https://worldofplayers.ru) [Worldofplayers (https://worldofplayers.ru)](https://worldofplayers.ru)*: top 10M, forum, ru* -1. ![](https://www.google.com/s2/favicons?domain=https://www.prizyvnik.info) [Prizyvnik (https://www.prizyvnik.info)](https://www.prizyvnik.info)*: top 10M, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://www.prizyvnik.info) [Prizyvnik (https://www.prizyvnik.info)](https://www.prizyvnik.info)*: top 10M, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://defenceforumindia.com/) [DefenceForumIndia (https://defenceforumindia.com/)](https://defenceforumindia.com/)*: top 10M, forum, in, military* 1. ![](https://www.google.com/s2/favicons?domain=https://ro-ru.ru) [Ro-ru (https://ro-ru.ru)](https://ro-ru.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://zhyk.ru) [Zhyk (https://zhyk.ru)](https://zhyk.ru)*: top 10M, forum, ru*, search is disabled @@ -1018,10 +1024,10 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.wireclub.com) [Wireclub (https://www.wireclub.com)](https://www.wireclub.com)*: top 10M, in, tr, us* 1. ![](https://www.google.com/s2/favicons?domain=https://community.go365.com) [Go365 (https://community.go365.com)](https://community.go365.com)*: top 10M, us*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://social.tchncs.de/) [social.tchncs.de (https://social.tchncs.de/)](https://social.tchncs.de/)*: top 10M, de, in* -1. ![](https://www.google.com/s2/favicons?domain=https://www.rusforum.com/) [Rusforum (https://www.rusforum.com/)](https://www.rusforum.com/)*: top 10M, forum, in, pk, ru, ua* +1. ![](https://www.google.com/s2/favicons?domain=https://www.rusforum.com/) [Rusforum (https://www.rusforum.com/)](https://www.rusforum.com/)*: top 10M, forum, in, pk, ru, ua*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://gribnyemesta.unoforum.pro) [Gribnyemesta (https://gribnyemesta.unoforum.pro)](https://gribnyemesta.unoforum.pro)*: top 10M, forum, ru* 1. ![](https://www.google.com/s2/favicons?domain=http://cheat-master.ru) [cheat-master.ru (http://cheat-master.ru)](http://cheat-master.ru)*: top 10M, ru, ua* -1. ![](https://www.google.com/s2/favicons?domain=https://lovetalk.ru) [Mylove (https://lovetalk.ru)](https://lovetalk.ru)*: top 10M, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://lovetalk.ru) [Mylove (https://lovetalk.ru)](https://lovetalk.ru)*: top 10M, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://board.german242.com) [German242 (https://board.german242.com)](https://board.german242.com)*: top 10M, forum, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://forum.beyond3d.com) [beyond3d (https://forum.beyond3d.com)](https://forum.beyond3d.com)*: top 10M, forum, in, pk, ru, us* 1. ![](https://www.google.com/s2/favicons?domain=https://lightstalking.us/) [Lightstalking (https://lightstalking.us/)](https://lightstalking.us/)*: top 10M, forum, photo* @@ -1049,7 +1055,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://partyflock.nl) [Partyflock (https://partyflock.nl)](https://partyflock.nl)*: top 10M, in, nl* 1. ![](https://www.google.com/s2/favicons?domain=http://forum.bratsk.org) [Bratsk Forum (http://forum.bratsk.org)](http://forum.bratsk.org)*: top 10M, forum, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://armtorg.ru/) [Armtorg (https://armtorg.ru/)](https://armtorg.ru/)*: top 10M, forum, ru* -1. ![](https://www.google.com/s2/favicons?domain=https://www.duno.com/) [Duno (https://www.duno.com/)](https://www.duno.com/)*: top 10M, in, pk* +1. ![](https://www.google.com/s2/favicons?domain=https://www.duno.com/) [Duno (https://www.duno.com/)](https://www.duno.com/)*: top 10M, in, pk*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.damochka.ru) [Damochka (https://www.damochka.ru)](https://www.damochka.ru)*: top 10M, kz, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=http://player.ru) [Player (http://player.ru)](http://player.ru)*: top 10M, forum, ru, shopping* 1. ![](https://www.google.com/s2/favicons?domain=https://python.su/) [python.su (https://python.su/)](https://python.su/)*: top 10M, ru* @@ -1089,12 +1095,12 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://forum.openframeworks.cc) [openframeworks (https://forum.openframeworks.cc)](https://forum.openframeworks.cc)*: top 10M, forum, us* 1. ![](https://www.google.com/s2/favicons?domain=http://sportsjournalists.com/forum/) [sportsjournalists.com (http://sportsjournalists.com/forum/)](http://sportsjournalists.com/forum/)*: top 10M, forum, us* 1. ![](https://www.google.com/s2/favicons?domain=https://www.gardening-forums.com) [Gardening-forums (https://www.gardening-forums.com)](https://www.gardening-forums.com)*: top 10M, forum, ph* -1. ![](https://www.google.com/s2/favicons?domain=https://forum.reverse4you.org) [reverse4you (https://forum.reverse4you.org)](https://forum.reverse4you.org)*: top 10M, forum, lk, ru, ua* +1. ![](https://www.google.com/s2/favicons?domain=https://forum.reverse4you.org) [reverse4you (https://forum.reverse4you.org)](https://forum.reverse4you.org)*: top 10M, forum, lk, ru, ua*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://macosx.com) [Macosx (https://macosx.com)](https://macosx.com)*: top 10M, forum* 1. ![](https://www.google.com/s2/favicons?domain=https://smashrun.com/) [Smashrun (https://smashrun.com/)](https://smashrun.com/)*: top 10M, br, jp, us* 1. ![](https://www.google.com/s2/favicons?domain=http://spishu.ru) [spishu.ru (http://spishu.ru)](http://spishu.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://www.thedaftclub.com) [Thedaftclub (https://www.thedaftclub.com)](https://www.thedaftclub.com)*: top 10M, us*, search is disabled -1. ![](https://www.google.com/s2/favicons?domain=https://forum.allgaz.ru) [allgaz (https://forum.allgaz.ru)](https://forum.allgaz.ru)*: top 10M, forum, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://forum.allgaz.ru) [allgaz (https://forum.allgaz.ru)](https://forum.allgaz.ru)*: top 10M, forum, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://forums.vitalfootball.co.uk) [VitalFootball (https://forums.vitalfootball.co.uk)](https://forums.vitalfootball.co.uk)*: top 10M, forum, gb, in, pk* 1. ![](https://www.google.com/s2/favicons?domain=https://vlmi.biz) [Vlmi (https://vlmi.biz)](https://vlmi.biz)*: top 10M, forum, ru, ua* 1. ![](https://www.google.com/s2/favicons?domain=https://www.sciax2.it/forum/) [sciax2.it (https://www.sciax2.it/forum/)](https://www.sciax2.it/forum/)*: top 10M, forum, tr* @@ -1131,7 +1137,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=http://forum.evaveda.com/) [ForumEvaveda (http://forum.evaveda.com/)](http://forum.evaveda.com/)*: top 10M, forum, ru* 1. ![](https://www.google.com/s2/favicons?domain=http://kpyto.pp.net.ua) [kpyto.pp.net.ua (http://kpyto.pp.net.ua)](http://kpyto.pp.net.ua)*: top 10M, ua* 1. ![](https://www.google.com/s2/favicons?domain=http://samsungmobile.pp.net.ua) [samsungmobile.pp.net.ua (http://samsungmobile.pp.net.ua)](http://samsungmobile.pp.net.ua)*: top 10M, ua* -1. ![](https://www.google.com/s2/favicons?domain=http://fapforum.net) [Fapforum (http://fapforum.net)](http://fapforum.net)*: top 10M, forum, porn, ua* +1. ![](https://www.google.com/s2/favicons?domain=http://fapforum.net) [Fapforum (http://fapforum.net)](http://fapforum.net)*: top 10M, forum, porn, ua*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://social.librem.one) [SocialLibremOne (https://social.librem.one)](https://social.librem.one)*: top 10M, tech* 1. ![](https://www.google.com/s2/favicons?domain=https://forums.footballindex.co.uk) [footballindex (https://forums.footballindex.co.uk)](https://forums.footballindex.co.uk)*: top 10M, forum, gb, in, sg, us* 1. ![](https://www.google.com/s2/favicons?domain=http://www.free-lancers.net) [Free-lancers (http://www.free-lancers.net)](http://www.free-lancers.net)*: top 10M, freelance, ru* @@ -1147,10 +1153,9 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=http://www.cigarpass.com/forum) [cigarpass.com (http://www.cigarpass.com/forum)](http://www.cigarpass.com/forum)*: top 10M, forum, ir* 1. ![](https://www.google.com/s2/favicons?domain=https://letschatlove.com) [Letschatlove (https://letschatlove.com)](https://letschatlove.com)*: top 10M, in* 1. ![](https://www.google.com/s2/favicons?domain=https://www.connosr.com/) [Connosr (https://www.connosr.com/)](https://www.connosr.com/)*: top 10M, gb* -1. ![](https://www.google.com/s2/favicons?domain=https://www.bayoushooter.com) [Bayoushooter (https://www.bayoushooter.com)](https://www.bayoushooter.com)*: top 10M, forum, pk, us* +1. ![](https://www.google.com/s2/favicons?domain=https://www.bayoushooter.com) [Bayoushooter (https://www.bayoushooter.com)](https://www.bayoushooter.com)*: top 10M, forum, pk, us*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://nygunforum.com) [Nygunforum (https://nygunforum.com)](https://nygunforum.com)*: top 10M, forum, us* 1. ![](https://www.google.com/s2/favicons?domain=http://www.astra-club.ru) [Astra-club (http://www.astra-club.ru)](http://www.astra-club.ru)*: top 10M, ru, ua*, search is disabled -1. ![](https://www.google.com/s2/favicons?domain=https://likee.com) [Likee (https://likee.com)](https://likee.com)*: top 10M, video* 1. ![](https://www.google.com/s2/favicons?domain=http://phrack.org) [Phrack (http://phrack.org)](http://phrack.org)*: top 10M* 1. ![](https://www.google.com/s2/favicons?domain=http://esate.ru) [Esate (http://esate.ru)](http://esate.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://ieoc.com/) [Ieoc (https://ieoc.com/)](https://ieoc.com/)*: top 10M, us* @@ -1185,7 +1190,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://www.pitomec.ru) [Pitomec (https://www.pitomec.ru)](https://www.pitomec.ru)*: top 10M, ru, ua* 1. ![](https://www.google.com/s2/favicons?domain=http://autokadabra.ru/) [Autokadabra (http://autokadabra.ru/)](http://autokadabra.ru/)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://movie-forum.co) [Movie-forum (https://movie-forum.co)](https://movie-forum.co)*: top 10M, forum, pk* -1. ![](https://www.google.com/s2/favicons?domain=http://peopleandcountries.com) [PeopleAndCountries (http://peopleandcountries.com)](http://peopleandcountries.com)*: top 10M, forum, ru* +1. ![](https://www.google.com/s2/favicons?domain=http://peopleandcountries.com) [PeopleAndCountries (http://peopleandcountries.com)](http://peopleandcountries.com)*: top 10M, forum, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=http://33bru.com/) [33bru (http://33bru.com/)](http://33bru.com/)*: top 10M, ru, ua* 1. ![](https://www.google.com/s2/favicons?domain=https://astrogalaxy.ru) [Astrogalaxy (https://astrogalaxy.ru)](https://astrogalaxy.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://maccentre.ru) [Maccentre (https://maccentre.ru)](https://maccentre.ru)*: top 10M, ru* @@ -1197,7 +1202,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://lab.pentestit.ru/) [labpentestit (https://lab.pentestit.ru/)](https://lab.pentestit.ru/)*: top 10M, hacking, ru* 1. ![](https://www.google.com/s2/favicons?domain=http://dieselmastera.ru) [dieselmastera.ru (http://dieselmastera.ru)](http://dieselmastera.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://forum.avtolyubiteli.com) [Avtolyubiteli (https://forum.avtolyubiteli.com)](https://forum.avtolyubiteli.com)*: top 10M, forum, ru* -1. ![](https://www.google.com/s2/favicons?domain=https://www.2fast4u.be) [2fast4u (https://www.2fast4u.be)](https://www.2fast4u.be)*: top 10M, nl* +1. ![](https://www.google.com/s2/favicons?domain=https://www.2fast4u.be) [2fast4u (https://www.2fast4u.be)](https://www.2fast4u.be)*: top 10M, nl*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://www.livetrack24.com) [LiveTrack24 (https://www.livetrack24.com)](https://www.livetrack24.com)*: top 10M* 1. ![](https://www.google.com/s2/favicons?domain=http://chemistlab.ru) [Chemistlab (http://chemistlab.ru)](http://chemistlab.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=http://upbyte.net) [upbyte.net (http://upbyte.net)](http://upbyte.net)*: top 10M, ru* @@ -1213,7 +1218,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://shotbow.net) [Shotbow (https://shotbow.net)](https://shotbow.net)*: top 10M, ca, forum, us*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://otechie.com) [Otechie (https://otechie.com)](https://otechie.com)*: top 10M, finance, us* 1. ![](https://www.google.com/s2/favicons?domain=http://www.volgogradru.com) [Volgogradru (http://www.volgogradru.com)](http://www.volgogradru.com)*: top 10M, ru* -1. ![](https://www.google.com/s2/favicons?domain=https://pilguy.com) [Pilguy (https://pilguy.com)](https://pilguy.com)*: top 10M, forum, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://pilguy.com) [Pilguy (https://pilguy.com)](https://pilguy.com)*: top 10M, forum, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://fullhub.ru/) [Fullhub (https://fullhub.ru/)](https://fullhub.ru/)*: top 10M, ru, ua* 1. ![](https://www.google.com/s2/favicons?domain=https://forum-b.ru) [forum-b.ru (https://forum-b.ru)](https://forum-b.ru)*: top 10M, forum, freelance, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://honda.org.ua) [Honda (https://honda.org.ua)](https://honda.org.ua)*: top 10M, ru, ua* @@ -1272,7 +1277,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=http://uaodessa.com) [uaodessa.com (http://uaodessa.com)](http://uaodessa.com)*: top 10M* 1. ![](https://www.google.com/s2/favicons?domain=http://www.oilcareer.ru) [Oilcareer (http://www.oilcareer.ru)](http://www.oilcareer.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=http://dr-denisov.ru) [dr-denisov.ru (http://dr-denisov.ru)](http://dr-denisov.ru)*: top 10M* -1. ![](https://www.google.com/s2/favicons?domain=http://fifasoccer.ru) [Fifasoccer (http://fifasoccer.ru)](http://fifasoccer.ru)*: top 10M, forum, ru, ua* +1. ![](https://www.google.com/s2/favicons?domain=http://fifasoccer.ru) [Fifasoccer (http://fifasoccer.ru)](http://fifasoccer.ru)*: top 10M, forum, ru, ua*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=https://forum.biohack.me) [biohack (https://forum.biohack.me)](https://forum.biohack.me)*: top 10M, forum* 1. ![](https://www.google.com/s2/favicons?domain=http://free-pass.ru) [free-pass.ru (http://free-pass.ru)](http://free-pass.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://forum.md/ru/) [md (https://forum.md/ru/)](https://forum.md/ru/)*: top 10M, forum, md, ru* @@ -1355,7 +1360,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://brute.su) [Brute (https://brute.su)](https://brute.su)*: top 10M, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=http://p8ntballer-forums.com/) [p8ntballer-forums.com (http://p8ntballer-forums.com/)](http://p8ntballer-forums.com/)*: top 10M, forum* 1. ![](https://www.google.com/s2/favicons?domain=http://memory57.ucoz.ru) [memory57.ucoz.ru (http://memory57.ucoz.ru)](http://memory57.ucoz.ru)*: top 10M* -1. ![](https://www.google.com/s2/favicons?domain=https://plug.dj/) [Plug.DJ (https://plug.dj/)](https://plug.dj/)*: top 10M, fr, gb, kr, us* +1. ![](https://www.google.com/s2/favicons?domain=https://plug.dj/) [Plug.DJ (https://plug.dj/)](https://plug.dj/)*: top 10M, music*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=http://zapravdu.ru/) [Zapravdu (http://zapravdu.ru/)](http://zapravdu.ru/)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=http://ramta.0pk.ru) [Ramta (http://ramta.0pk.ru)](http://ramta.0pk.ru)*: top 10M, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://mediarepost.ru) [Mediarepost (https://mediarepost.ru)](https://mediarepost.ru)*: top 10M, ru* @@ -1407,7 +1412,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=http://diz-cs.ru) [diz-cs.ru (http://diz-cs.ru)](http://diz-cs.ru)*: top 10M* 1. ![](https://www.google.com/s2/favicons?domain=https://g-news.com.ua) [G-news (https://g-news.com.ua)](https://g-news.com.ua)*: top 10M, in, ua*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=http://igra-online.ucoz.com) [igra-online.ucoz.com (http://igra-online.ucoz.com)](http://igra-online.ucoz.com)*: top 10M* -1. ![](https://www.google.com/s2/favicons?domain=https://joby.su) [Joby (https://joby.su)](https://joby.su)*: top 10M, freelance, ru* +1. ![](https://www.google.com/s2/favicons?domain=https://joby.su) [Joby (https://joby.su)](https://joby.su)*: top 10M, freelance, ru*, search is disabled 1. ![](https://www.google.com/s2/favicons?domain=http://forum.scuba-divers.ru/) [scuba (http://forum.scuba-divers.ru/)](http://forum.scuba-divers.ru/)*: top 10M, forum, ru* 1. ![](https://www.google.com/s2/favicons?domain=https://boominfo.ru) [BoomInfo (https://boominfo.ru)](https://boominfo.ru)*: top 10M, ru, ua* 1. ![](https://www.google.com/s2/favicons?domain=http://chastysc.ucoz.ru) [chastysc.ucoz.ru (http://chastysc.ucoz.ru)](http://chastysc.ucoz.ru)*: top 10M, ru* @@ -2531,6 +2536,12 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=http://weebly.com) [Weebly (http://weebly.com)](http://weebly.com)*: top 100M, business* 1. ![](https://www.google.com/s2/favicons?domain=http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion) [HiddenAnswers (http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion)](http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion)*: top 100M, tor* 1. ![](https://www.google.com/s2/favicons?domain={username}.com) [.com ({username}.com)]({username}.com)*: top 100M* +1. ![](https://www.google.com/s2/favicons?domain={username}.pro) [.pro ({username}.pro)]({username}.pro)*: top 100M* +1. ![](https://www.google.com/s2/favicons?domain={username}.me) [.me ({username}.me)]({username}.me)*: top 100M* +1. ![](https://www.google.com/s2/favicons?domain={username}.biz) [.biz ({username}.biz)]({username}.biz)*: top 100M* +1. ![](https://www.google.com/s2/favicons?domain={username}.email) [.email ({username}.email)]({username}.email)*: top 100M* +1. ![](https://www.google.com/s2/favicons?domain={username}.guru) [.guru ({username}.guru)]({username}.guru)*: top 100M* +1. ![](https://www.google.com/s2/favicons?domain={username}.ddns.net) [.ddns.net ({username}.ddns.net)]({username}.ddns.net)*: top 100M* 1. ![](https://www.google.com/s2/favicons?domain=https://galactictalk.org) [galactictalk.org (https://galactictalk.org)](https://galactictalk.org)*: top 100M* 1. ![](https://www.google.com/s2/favicons?domain=https://discuss.bootstrapped.fm) [discuss.bootstrapped.fm (https://discuss.bootstrapped.fm)](https://discuss.bootstrapped.fm)*: top 100M* 1. ![](https://www.google.com/s2/favicons?domain=https://discourse.mozilla.org) [discourse.mozilla.org (https://discourse.mozilla.org)](https://discourse.mozilla.org)*: top 100M* @@ -2563,5 +2574,8 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://rec.poker) [rec.poker (https://rec.poker)](https://rec.poker)*: top 100M* 1. ![](https://www.google.com/s2/favicons?domain=https://uforum.uz) [uforum.uz (https://uforum.uz)](https://uforum.uz)*: top 100M* 1. ![](https://www.google.com/s2/favicons?domain=http://dntrustmucd4mwec.onion) [DarkNet Trust (http://dntrustmucd4mwec.onion)](http://dntrustmucd4mwec.onion)*: top 100M, tor* +1. ![](https://www.google.com/s2/favicons?domain=http://i2pforum.i2p) [i2pforum (http://i2pforum.i2p)](http://i2pforum.i2p)*: top 100M, i2p* +1. ![](https://www.google.com/s2/favicons?domain=https://partnerkin.com) [partnerkin.com (https://partnerkin.com)](https://partnerkin.com)*: top 100M, finance* +1. ![](https://www.google.com/s2/favicons?domain=https://www.hozpitality.com) [hozpitality (https://www.hozpitality.com)](https://www.hozpitality.com)*: top 100M* -Alexa.com rank data fetched at (2021-05-31 21:26:56.886650 UTC) +Alexa.com rank data fetched at (2021-12-12 10:53:18.497003 UTC) From 744ad1deda8a7f0ab81b9e0bbb330fd09e2b6c14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 19:42:30 +0300 Subject: [PATCH 61/72] Bump lxml from 4.6.4 to 4.6.5 (#266) Bumps [lxml](https://github.com/lxml/lxml) from 4.6.4 to 4.6.5. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.4...lxml-4.6.5) --- updated-dependencies: - dependency-name: lxml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 47c52918b..131b158d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ future-annotations==1.0.0 html5lib==1.1 idna==3.3 Jinja2==3.0.3 -lxml==4.6.4 +lxml==4.6.5 MarkupSafe==2.0.1 mock==4.0.3 multidict==5.2.0 From 56ad1d1c606c9c35c3eff702d8dab36d763151b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Dec 2021 09:41:24 +0300 Subject: [PATCH 62/72] Bump lxml from 4.6.5 to 4.7.1 (#269) Bumps [lxml](https://github.com/lxml/lxml) from 4.6.5 to 4.7.1. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.5...lxml-4.7.1) --- updated-dependencies: - dependency-name: lxml dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 131b158d5..587cfecb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ future-annotations==1.0.0 html5lib==1.1 idna==3.3 Jinja2==3.0.3 -lxml==4.6.5 +lxml==4.7.1 MarkupSafe==2.0.1 mock==4.0.3 multidict==5.2.0 From b7816024740d49cc2d3739884f0d1f3f27dedec1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Dec 2021 21:47:49 +0300 Subject: [PATCH 63/72] Bump pytest-httpserver from 1.0.2 to 1.0.3 (#270) Bumps [pytest-httpserver](https://github.com/csernazs/pytest-httpserver) from 1.0.2 to 1.0.3. - [Release notes](https://github.com/csernazs/pytest-httpserver/releases) - [Changelog](https://github.com/csernazs/pytest-httpserver/blob/master/CHANGES.rst) - [Commits](https://github.com/csernazs/pytest-httpserver/compare/1.0.2...1.0.3) --- updated-dependencies: - dependency-name: pytest-httpserver dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 9a0bafa64..c16fac101 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,5 +2,5 @@ flake8==4.0.1 pytest==6.2.5 pytest-asyncio==0.16.0 pytest-cov==3.0.0 -pytest-httpserver==1.0.2 +pytest-httpserver==1.0.3 pytest-rerunfailures==10.2 From bda85b290ea7d26c86e162484c2bf3335b76e2a0 Mon Sep 17 00:00:00 2001 From: Soxoj <31013580+soxoj@users.noreply.github.com> Date: Mon, 20 Dec 2021 23:16:33 +0300 Subject: [PATCH 64/72] Fixed failed tests (thx to Meta aka Facebook) (#273) --- maigret/resources/data.json | 2 +- tests/test_maigret.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 82c5c889c..2fb5e582a 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -6100,7 +6100,7 @@ "alexaRank": 1, "urlMain": "https://play.google.com/store", "url": "https://play.google.com/store/apps/developer?id={username}", - "usernameClaimed": "Facebook", + "usernameClaimed": "Skyeng", "usernameUnclaimed": "noonewouldeverusethis7" }, "Gorod.dp.ua": { diff --git a/tests/test_maigret.py b/tests/test_maigret.py index 0c05262cf..c61f5ffc2 100644 --- a/tests/test_maigret.py +++ b/tests/test_maigret.py @@ -19,7 +19,7 @@ 'cookies': None, 'parsing_enabled': False, 'url_main': 'https://www.reddit.com/', - 'username': 'Facebook', + 'username': 'Skyeng', }, 'GooglePlayStore': { 'cookies': None, @@ -28,8 +28,8 @@ 'parsing_enabled': False, 'rank': 1, 'url_main': 'https://play.google.com/store', - 'url_user': 'https://play.google.com/store/apps/developer?id=Facebook', - 'username': 'Facebook', + 'url_user': 'https://play.google.com/store/apps/developer?id=Skyeng', + 'username': 'Skyeng', }, } @@ -52,7 +52,7 @@ def test_self_check_db_positive_enable(test_db): logger = Mock() test_db.sites[0].disabled = True - test_db.sites[0].username_claimed = 'Facebook' + test_db.sites[0].username_claimed = 'Skyeng' assert test_db.sites[0].disabled is True loop = asyncio.get_event_loop() @@ -83,7 +83,7 @@ def test_self_check_db_negative_enabled(test_db): logger = Mock() test_db.sites[0].disabled = False - test_db.sites[0].username_claimed = 'Facebook' + test_db.sites[0].username_claimed = 'Skyeng' assert test_db.sites[0].disabled is False loop = asyncio.get_event_loop() @@ -98,7 +98,7 @@ def test_self_check_db_negative_enabled(test_db): def test_maigret_results(test_db): logger = Mock() - username = 'Facebook' + username = 'Skyeng' loop = asyncio.get_event_loop() results = loop.run_until_complete( maigret(username, site_dict=test_db.sites_dict, logger=logger, timeout=30) From c5df7ca99040ee4e754ddf7eb187339bc3cc4570 Mon Sep 17 00:00:00 2001 From: Soxoj <31013580+soxoj@users.noreply.github.com> Date: Tue, 21 Dec 2021 23:54:21 +0300 Subject: [PATCH 65/72] Fixed votetags, updated issue template (#278) --- .github/ISSUE_TEMPLATE/report-false-result.md | 10 ++++++++++ maigret/resources/data.json | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/report-false-result.md b/.github/ISSUE_TEMPLATE/report-false-result.md index fae9cd7e0..f757dcbf0 100644 --- a/.github/ISSUE_TEMPLATE/report-false-result.md +++ b/.github/ISSUE_TEMPLATE/report-false-result.md @@ -8,3 +8,13 @@ assignees: soxoj --- Invalid link: + + + +- [ ] I'm sure that the link leads to "not found" page diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 2fb5e582a..dbaf08842 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -15106,7 +15106,8 @@ "checkType": "message", "absenceStrs": [ " looking for. Perhaps searching can help.", - "" + "", + "Page not found" ], "alexaRank": 39522, "urlMain": "https://www.votetags.info/", From bc8e29e92a89a6905b2042d3e033d43e3404a5b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Dec 2021 23:54:30 +0300 Subject: [PATCH 66/72] Bump async-timeout from 4.0.1 to 4.0.2 (#275) Bumps [async-timeout](https://github.com/aio-libs/async-timeout) from 4.0.1 to 4.0.2. - [Release notes](https://github.com/aio-libs/async-timeout/releases) - [Changelog](https://github.com/aio-libs/async-timeout/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/async-timeout/compare/v4.0.1...v4.0.2) --- updated-dependencies: - dependency-name: async-timeout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 587cfecb9..7dad5b22d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ aiodns==3.0.0 aiohttp==3.8.1 aiohttp-socks==0.7.1 arabic-reshaper==2.1.3 -async-timeout==4.0.1 +async-timeout==4.0.2 attrs==21.2.0 certifi==2021.10.8 chardet==4.0.0 From 1dfc45722bd94ec069a1513ee03f94ea686e9684 Mon Sep 17 00:00:00 2001 From: Soxoj <31013580+soxoj@users.noreply.github.com> Date: Sun, 26 Dec 2021 19:47:26 +0300 Subject: [PATCH 67/72] Fixed some false positives (#280) --- maigret/resources/data.json | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/maigret/resources/data.json b/maigret/resources/data.json index dbaf08842..884b20554 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -2178,7 +2178,8 @@ ], "checkType": "message", "absenceStrs": [ - "Sorry! We couldn\u2019t find what you were looking for" + "Sorry! We couldn\u2019t find what you were looking for", + "error-cartoon__image" ], "alexaRank": 88183, "urlMain": "https://ru.bookmate.com", @@ -2876,7 +2877,10 @@ "coding", "education" ], - "checkType": "status_code", + "checkType": "message", + "absenceStrs": [ + "This profile does not exist" + ], "alexaRank": 2566, "urlMain": "https://www.codecademy.com/", "url": "https://www.codecademy.com/profiles/{username}", @@ -6116,6 +6120,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Gorodanapa": { + "disabled": true, "tags": [ "ru" ], @@ -14211,7 +14216,8 @@ "urlMain": "https://tproger.ru", "url": "https://tproger.ru/author/{username}/", "usernameClaimed": "NickPrice", - "usernameUnclaimed": "noonewouldeverusethis7" + "usernameUnclaimed": "noonewouldeverusethis7", + "disabled": true }, "TrackmaniaLadder": { "tags": [ From 4ee65e0445da471b994359eeb348b92b16a23b90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Dec 2021 10:53:44 +0300 Subject: [PATCH 68/72] Bump attrs from 21.2.0 to 21.3.0 (#281) Bumps [attrs](https://github.com/python-attrs/attrs) from 21.2.0 to 21.3.0. - [Release notes](https://github.com/python-attrs/attrs/releases) - [Changelog](https://github.com/python-attrs/attrs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-attrs/attrs/compare/21.2.0...21.3.0) --- updated-dependencies: - dependency-name: attrs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7dad5b22d..4e490747f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ aiohttp==3.8.1 aiohttp-socks==0.7.1 arabic-reshaper==2.1.3 async-timeout==4.0.2 -attrs==21.2.0 +attrs==21.3.0 certifi==2021.10.8 chardet==4.0.0 colorama==0.4.4 From d52ff10186573fcb0de7be24b8f2c718dee542fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Dec 2021 10:29:04 +0300 Subject: [PATCH 69/72] Bump psutil from 5.8.0 to 5.9.0 (#282) Bumps [psutil](https://github.com/giampaolo/psutil) from 5.8.0 to 5.9.0. - [Release notes](https://github.com/giampaolo/psutil/releases) - [Changelog](https://github.com/giampaolo/psutil/blob/master/HISTORY.rst) - [Commits](https://github.com/giampaolo/psutil/compare/release-5.8.0...release-5.9.0) --- updated-dependencies: - dependency-name: psutil dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyinstaller/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyinstaller/requirements.txt b/pyinstaller/requirements.txt index 1fb924a16..81a5672b5 100644 --- a/pyinstaller/requirements.txt +++ b/pyinstaller/requirements.txt @@ -1,5 +1,5 @@ maigret @ https://github.com/soxoj/maigret/archive/refs/heads/main.zip pefile==2021.9.3 -psutil==5.8.0 +psutil==5.9.0 pyinstaller @ https://github.com/pyinstaller/pyinstaller/archive/develop.zip pywin32-ctypes==0.2.0 \ No newline at end of file From 8801f7e6de404f616ce34bc367fc86fb16170e29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Dec 2021 10:29:12 +0300 Subject: [PATCH 70/72] Bump attrs from 21.3.0 to 21.4.0 (#283) Bumps [attrs](https://github.com/python-attrs/attrs) from 21.3.0 to 21.4.0. - [Release notes](https://github.com/python-attrs/attrs/releases) - [Changelog](https://github.com/python-attrs/attrs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-attrs/attrs/compare/21.3.0...21.4.0) --- updated-dependencies: - dependency-name: attrs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4e490747f..01e30ed07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ aiohttp==3.8.1 aiohttp-socks==0.7.1 arabic-reshaper==2.1.3 async-timeout==4.0.2 -attrs==21.3.0 +attrs==21.4.0 certifi==2021.10.8 chardet==4.0.0 colorama==0.4.4 From ecabf88c3a10054e38fa25ba7dc0d63310ffd3ed Mon Sep 17 00:00:00 2001 From: Soxoj <31013580+soxoj@users.noreply.github.com> Date: Mon, 3 Jan 2022 01:35:53 +0300 Subject: [PATCH 71/72] Added a couple of sites, fixed false positives (#286) --- maigret/maigret.py | 2 +- maigret/resources/data.json | 152 ++++++++++++++++++++++++++++++++++++ maigret/submit.py | 57 ++++++++------ utils/update_site_data.py | 2 +- 4 files changed, 189 insertions(+), 24 deletions(-) diff --git a/maigret/maigret.py b/maigret/maigret.py index a72274087..75b0c7fb9 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -536,7 +536,7 @@ async def main(): site_data = get_top_sites_for_id(args.id_type) if args.new_site_to_submit: - submitter = Submitter(db=db, logger=logger, settings=settings) + submitter = Submitter(db=db, logger=logger, settings=settings, args=args) is_submitted = await submitter.dialog(args.new_site_to_submit, args.cookie_file) if is_submitted: db.save_to_file(db_file) diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 884b20554..e212bda33 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -1833,6 +1833,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Bestfantasybooks": { + "disabled": true, "tags": [ "us" ], @@ -4432,6 +4433,7 @@ ] }, "Facenama": { + "disabled": true, "tags": [ "ir" ], @@ -28440,6 +28442,156 @@ "usernameUnclaimed": "noonewouldeverusethis7", "checkType": "message", "alexaRank": 6859 + }, + "Worldis.me": { + "absenceStrs": [ + "user_password", + "send_email" + ], + "presenseStrs": [ + "my_profile", + "profile_upi", + "UserInfo" + ], + "url": "http://en.worldis.me/{username}", + "urlMain": "http://en.worldis.me", + "usernameClaimed": "admin", + "usernameUnclaimed": "noonewouldeverusethis7", + "checkType": "message", + "alexaRank": 3233509, + "tags": [ + "ru" + ] + }, + "photoshop-kopona.com": { + "absenceStrs": [ + "<title>noonewouldeverusethis7 » \u0420\u0435\u0441\u0443\u0440\u0441\u044b \u0434\u043b\u044f \u0424\u043e\u0442\u043e\u0448\u043e\u043f\u0430" + ], + "presenseStrs": [ + "offline", + "uspusertitle" + ], + "url": "https://photoshop-kopona.com/ru/user/{username}/", + "urlMain": "https://photoshop-kopona.com", + "usernameClaimed": "test", + "usernameUnclaimed": "noonewouldeverusethis7", + "checkType": "message", + "alexaRank": 44106, + "tags": [ + "ru" + ] + }, + "dumskaya.net": { + "absenceStrs": [ + "> + ], + + ], + Username:" + ], + "presenseStrs": [ + "email", + "usernamereg", + "username-top", + "\u041e\u043f\u044b\u0442 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f", + "check-username" + ], + "url": "https://xgm.guru/user/{username}", + "urlMain": "https://xgm.guru", + "usernameClaimed": "test", + "usernameUnclaimed": "noonewouldeverusethis7", + "checkType": "message", + "alexaRank": 692341, + "tags": [ + "forum", + "gaming" + ] } }, "engines": { diff --git a/maigret/submit.py b/maigret/submit.py index 7c4216ba2..352946fea 100644 --- a/maigret/submit.py +++ b/maigret/submit.py @@ -3,6 +3,7 @@ import re from typing import List import xml.etree.ElementTree as ET +from aiohttp import TCPConnector, ClientSession import requests from .activation import import_aiohttp_cookies @@ -24,11 +25,24 @@ class Submitter: TOP_FEATURES = 5 URL_RE = re.compile(r"https?://(www\.)?") - def __init__(self, db: MaigretDatabase, settings: Settings, logger): + def __init__(self, db: MaigretDatabase, settings: Settings, logger, args): self.settings = settings + self.args = args self.db = db self.logger = logger + from aiohttp_socks import ProxyConnector + proxy = self.args.proxy + cookie_jar = None + if args.cookie_file: + cookie_jar = import_aiohttp_cookies(args.cookie_file) + + connector = ProxyConnector.from_url(proxy) if proxy else TCPConnector(ssl=False) + connector.verify_ssl = False + self.session = ClientSession( + connector=connector, trust_env=True, cookie_jar=cookie_jar + ) + @staticmethod def get_alexa_rank(site_url_main): url = f"http://data.alexa.com/data?cli=10&url={site_url_main}" @@ -63,6 +77,7 @@ async def site_self_check(self, site, semaphore, silent=False): results_dict = await maigret( username=username, site_dict={site.name: site}, + proxy=self.args.proxy, logger=self.logger, timeout=30, id_type=site.type, @@ -126,9 +141,11 @@ def generate_additional_fields_dialog(self, engine: MaigretEngine, dialog): return fields async def detect_known_engine(self, url_exists, url_mainpage) -> List[MaigretSite]: + resp_text = '' try: - r = requests.get(url_mainpage) - self.logger.debug(r.text) + r = await self.session.get(url_mainpage) + resp_text = await r.text() + self.logger.debug(resp_text) except Exception as e: self.logger.warning(e) print("Some error while checking main page") @@ -136,10 +153,10 @@ async def detect_known_engine(self, url_exists, url_mainpage) -> List[MaigretSit for engine in self.db.engines: strs_to_check = engine.__dict__.get("presenseStrs") - if strs_to_check and r and r.text: + if strs_to_check and resp_text: all_strs_in_response = True for s in strs_to_check: - if s not in r.text: + if s not in resp_text: all_strs_in_response = False sites = [] if all_strs_in_response: @@ -209,32 +226,28 @@ async def check_features_manually( headers = dict(self.HEADERS) headers.update(custom_headers) - # cookies - cookie_dict = None - if cookie_file: - self.logger.info(f'Use {cookie_file} for cookies') - cookie_jar = import_aiohttp_cookies(cookie_file) - cookie_dict = {c.key: c.value for c in cookie_jar} - - exists_resp = requests.get( - url_exists, cookies=cookie_dict, headers=headers, allow_redirects=redirects + exists_resp = await self.session.get( + url_exists, + headers=headers, + allow_redirects=redirects, ) + exists_resp_text = await exists_resp.text() self.logger.debug(url_exists) - self.logger.debug(exists_resp.status_code) - self.logger.debug(exists_resp.text) + self.logger.debug(exists_resp.status) + self.logger.debug(exists_resp_text) - non_exists_resp = requests.get( + non_exists_resp = await self.session.get( url_not_exists, - cookies=cookie_dict, headers=headers, allow_redirects=redirects, ) + non_exists_resp_text = await non_exists_resp.text() self.logger.debug(url_not_exists) - self.logger.debug(non_exists_resp.status_code) - self.logger.debug(non_exists_resp.text) + self.logger.debug(non_exists_resp.status) + self.logger.debug(non_exists_resp_text) - a = exists_resp.text - b = non_exists_resp.text + a = exists_resp_text + b = non_exists_resp_text tokens_a = set(re.split(f'[{self.SEPARATORS}]', a)) tokens_b = set(re.split(f'[{self.SEPARATORS}]', b)) diff --git a/utils/update_site_data.py b/utils/update_site_data.py index 8683255ac..12180fbaf 100755 --- a/utils/update_site_data.py +++ b/utils/update_site_data.py @@ -25,7 +25,7 @@ '100000000': '100M', }) -SEMAPHORE = threading.Semaphore(10) +SEMAPHORE = threading.Semaphore(20) def get_rank(domain_to_query, site, print_errors=True): with SEMAPHORE: From e520418f6a25d7edacde2d73b41a8ae7c80ddf39 Mon Sep 17 00:00:00 2001 From: Soxoj <31013580+soxoj@users.noreply.github.com> Date: Mon, 3 Jan 2022 01:44:11 +0300 Subject: [PATCH 72/72] Bump to v0.4.0 (#287) --- CHANGELOG.md | 67 ++++++++++++++++++++++++++++++++++++++++++ docs/source/conf.py | 4 +-- maigret/__version__.py | 2 +- setup.py | 2 +- 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bda589bcc..c1c76e472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,73 @@ ## [Unreleased] +## [0.4.0] - 2022-01-03 +* Delayed import of requests module, speed check command, reqs updated by @soxoj in https://github.com/soxoj/maigret/pull/189 +* Snapcraft yaml added by @soxoj in https://github.com/soxoj/maigret/pull/190 +* Create codeql-analysis.yml by @soxoj in https://github.com/soxoj/maigret/pull/191 +* Move wiki pages to ReadTheDocs by @egornagornov in https://github.com/soxoj/maigret/pull/194 +* Created ReadTheDocs requirements file by @soxoj in https://github.com/soxoj/maigret/pull/195 +* Fix incompatible version requirements by @JasperJuergensen in https://github.com/soxoj/maigret/pull/196 +* Added link to documentation by @soxoj in https://github.com/soxoj/maigret/pull/198 +* Upgraded base docker image by @soxoj in https://github.com/soxoj/maigret/pull/199 +* Run CodeQL only aflter merge and each Saturday by @soxoj in https://github.com/soxoj/maigret/pull/201 +* Added cascade settings loading from /.maigret/settings.json and ./settings.json by @soxoj in https://github.com/soxoj/maigret/pull/200 +* Documentation and settings improved by @soxoj in https://github.com/soxoj/maigret/pull/203 +* New config options added by @soxoj in https://github.com/soxoj/maigret/pull/204 +* Added export of cli entrypoint by @soxoj in https://github.com/soxoj/maigret/pull/207 +* Removed redundant logging by @soxoj in https://github.com/soxoj/maigret/pull/210 +* PyInstaller workflow by @soxoj in https://github.com/soxoj/maigret/pull/206 +* Create bug.md by @soxoj in https://github.com/soxoj/maigret/pull/213 +* Fixed path and names of report files by @soxoj in https://github.com/soxoj/maigret/pull/216 +* Box drawing logic improved, added new settings by @soxoj in https://github.com/soxoj/maigret/pull/217 +* Fixes for win32 release by @soxoj in https://github.com/soxoj/maigret/pull/218 +* Bump six from 1.15.0 to 1.16.0 by @dependabot in https://github.com/soxoj/maigret/pull/221 +* Bump flake8 from 3.8.4 to 4.0.1 by @dependabot in https://github.com/soxoj/maigret/pull/219 +* Bump aiohttp from 3.7.4 to 3.8.0 by @dependabot in https://github.com/soxoj/maigret/pull/220 +* Bump aiohttp-socks from 0.5.5 to 0.6.0 by @dependabot in https://github.com/soxoj/maigret/pull/222 +* Bump typing-extensions from 3.7.4.3 to 3.10.0.2 by @dependabot in https://github.com/soxoj/maigret/pull/224 +* Bump multidict from 5.1.0 to 5.2.0 by @dependabot in https://github.com/soxoj/maigret/pull/225 +* Bump idna from 2.10 to 3.3 by @dependabot in https://github.com/soxoj/maigret/pull/228 +* Bump pytest-cov from 2.10.1 to 3.0.0 by @dependabot in https://github.com/soxoj/maigret/pull/227 +* Bump mock from 4.0.2 to 4.0.3 by @dependabot in https://github.com/soxoj/maigret/pull/226 +* Bump certifi from 2020.12.5 to 2021.10.8 by @dependabot in https://github.com/soxoj/maigret/pull/233 +* Bump pytest-httpserver from 1.0.0 to 1.0.2 by @dependabot in https://github.com/soxoj/maigret/pull/232 +* Bump lxml from 4.6.3 to 4.6.4 by @dependabot in https://github.com/soxoj/maigret/pull/231 +* Bump pefile from 2019.4.18 to 2021.9.3 by @dependabot in https://github.com/soxoj/maigret/pull/229 +* Bump pytest-rerunfailures from 9.1.1 to 10.2 by @dependabot in https://github.com/soxoj/maigret/pull/230 +* Bump yarl from 1.6.3 to 1.7.2 by @dependabot in https://github.com/soxoj/maigret/pull/237 +* Bump async-timeout from 4.0.0 to 4.0.1 by @dependabot in https://github.com/soxoj/maigret/pull/236 +* Bump psutil from 5.7.0 to 5.8.0 by @dependabot in https://github.com/soxoj/maigret/pull/234 +* Bump jinja2 from 3.0.2 to 3.0.3 by @dependabot in https://github.com/soxoj/maigret/pull/235 +* Bump pytest from 6.2.4 to 6.2.5 by @dependabot in https://github.com/soxoj/maigret/pull/238 +* Bump tqdm from 4.55.0 to 4.62.3 by @dependabot in https://github.com/soxoj/maigret/pull/242 +* Bump arabic-reshaper from 2.1.1 to 2.1.3 by @dependabot in https://github.com/soxoj/maigret/pull/243 +* Bump pytest-asyncio from 0.14.0 to 0.16.0 by @dependabot in https://github.com/soxoj/maigret/pull/240 +* Bump chardet from 3.0.4 to 4.0.0 by @dependabot in https://github.com/soxoj/maigret/pull/241 +* Bump soupsieve from 2.1 to 2.3.1 by @dependabot in https://github.com/soxoj/maigret/pull/239 +* Bump aiohttp from 3.8.0 to 3.8.1 by @dependabot in https://github.com/soxoj/maigret/pull/246 +* Bump typing-extensions from 3.10.0.2 to 4.0.0 by @dependabot in https://github.com/soxoj/maigret/pull/245 +* Bump aiohttp-socks from 0.6.0 to 0.6.1 by @dependabot in https://github.com/soxoj/maigret/pull/249 +* Bump aiohttp-socks from 0.6.1 to 0.7.1 by @dependabot in https://github.com/soxoj/maigret/pull/250 +* Bump typing-extensions from 4.0.0 to 4.0.1 by @dependabot in https://github.com/soxoj/maigret/pull/253 +* Fixed some false positives by @soxoj in https://github.com/soxoj/maigret/pull/254 +* Disabled non-working sites by @soxoj in https://github.com/soxoj/maigret/pull/255 +* Added false results buttons to reports, fixed some falses by @soxoj in https://github.com/soxoj/maigret/pull/256 +* Fixed xHamster, added support of proxies to self-check mode by @soxoj in https://github.com/soxoj/maigret/pull/259 +* Disabled non-working sites, updated public sites list by @soxoj in https://github.com/soxoj/maigret/pull/263 +* Bump lxml from 4.6.4 to 4.6.5 by @dependabot in https://github.com/soxoj/maigret/pull/266 +* Bump lxml from 4.6.5 to 4.7.1 by @dependabot in https://github.com/soxoj/maigret/pull/269 +* Bump pytest-httpserver from 1.0.2 to 1.0.3 by @dependabot in https://github.com/soxoj/maigret/pull/270 +* Fixed failed tests (thx to Meta aka Facebook) by @soxoj in https://github.com/soxoj/maigret/pull/273 +* Fixed votetags, updated issue template by @soxoj in https://github.com/soxoj/maigret/pull/278 +* Bump async-timeout from 4.0.1 to 4.0.2 by @dependabot in https://github.com/soxoj/maigret/pull/275 +* Fixed some false positives by @soxoj in https://github.com/soxoj/maigret/pull/280 +* Bump attrs from 21.2.0 to 21.3.0 by @dependabot in https://github.com/soxoj/maigret/pull/281 +* Bump psutil from 5.8.0 to 5.9.0 by @dependabot in https://github.com/soxoj/maigret/pull/282 +* Bump attrs from 21.3.0 to 21.4.0 by @dependabot in https://github.com/soxoj/maigret/pull/283 + +**Full Changelog**: https://github.com/soxoj/maigret/compare/v0.3.1...v0.4.0 + ## [0.3.1] - 2021-10-31 * fixed false positives * accelerated maigret start time by 3 times diff --git a/docs/source/conf.py b/docs/source/conf.py index a0d4a62d6..9f7eab117 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -6,8 +6,8 @@ copyright = '2021, soxoj' author = 'soxoj' -release = '0.3.1' -version = '0.3.1' +release = '0.4.0' +version = '0.4.0' # -- General configuration diff --git a/maigret/__version__.py b/maigret/__version__.py index ab86bebf4..d43dffaef 100644 --- a/maigret/__version__.py +++ b/maigret/__version__.py @@ -1,3 +1,3 @@ """Maigret version file""" -__version__ = '0.3.1' +__version__ = '0.4.0' diff --git a/setup.py b/setup.py index 606174e62..f1541812f 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ requires = rf.read().splitlines() setup(name='maigret', - version='0.3.1', + version='0.4.0', description='Collect a dossier on a person by username from a huge number of sites', long_description=long_description, long_description_content_type="text/markdown",