geminispace.info

Unnamed repository; edit this file 'description' to name the repository.
git clone git://code.clttr.info/geminispace.info.git
Log | Files | Refs | README | LICENSE

commit 6e54e520144aea538eccf830b2985a41878f8899
parent c791a758f2e2804279ef964486b03854bf07b4a7
Author: René Wagner <rwagner@rw-net.de>
Date:   Mon,  8 Mar 2021 18:51:28 +0100

Merge branch 'master' of git://natpen.net/gus

Diffstat:
Mgus/crawl.py | 2++
1 file changed, 2 insertions(+), 0 deletions(-)

diff --git a/gus/crawl.py b/gus/crawl.py @@ -495,6 +495,8 @@ def crawl_page( # RobotFileParser will return the higher level value (*) if # no indexer section is found can_fetch = robots_file.can_fetch("indexer", gr.normalized_url) + + # same approach as above - last value wins crawl_delay = robots_file.crawl_delay("indexer") if not can_fetch: