geminispace.info

Unnamed repository; edit this file 'description' to name the repository.
git clone git://code.clttr.info/geminispace.info.git
Log | Files | Refs | README | LICENSE

commit 0657f5703ef85ab8672b4a800d12ffb8df713d61
parent fe33be1c978c66af467b16787b18e63672120acb
Author: Natalie Pendragon <natpen@natpen.net>
Date:   Sun, 15 Nov 2020 08:30:01 -0500

More exclusions

Diffstat:
Mgus/crawl.py | 8++++++++
1 file changed, 8 insertions(+), 0 deletions(-)

diff --git a/gus/crawl.py b/gus/crawl.py @@ -133,7 +133,15 @@ EXCLUDED_URL_PREFIXES = [ # issues in its content or something, but that's a problem for a # different day "gemini://gemini.spam.works/users/dvn/archive/", + "gemini://gem.denarii.cloud/chaindata.zip", + # full web proxy + "gemini://drewdevault.com/cgi-bin/web.sh?", + + # killing crawl, I think maybe because it's too big + "gemini://gem.denarii.cloud/pichaindata.zip", + + "gemini://matrix.kiwifarms.net", ] EXCLUDED_URL_PATHS = [