geminispace.info

Unnamed repository; edit this file 'description' to name the repository.
git clone git://code.clttr.info/geminispace.info.git
Log | Files | Refs | README | LICENSE

commit a4c8ddc92b939eb8d644ecc41d61f35e58e213c9
parent 80ad98435233fab6ae7642bb603353d01b577f2b
Author: Natalie Pendragon <natpen@natpen.net>
Date:   Sun, 23 Feb 2020 09:30:52 -0500

Add some new seed sites

Diffstat:
Mgus/crawl.py | 4+++-
1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gus/crawl.py b/gus/crawl.py @@ -23,6 +23,9 @@ SEED_URLS = [ "gemini://tilde.pink:1965", "gemini://vger.cloud:1965", "gemini://yam655.com:1965", + "gemini://dump.royniang.com", + "gemini://konpeito.media", + "gemini://gemini.68kmentat.com", ] @@ -86,7 +89,6 @@ def get_robots_file(url): if r is not None and r.status.startswith("2"): robot_file_parser = robotparser.RobotFileParser() robot_file_parser.parse(r.content) - print(robot_file_parser.site_maps()) robot_file_map[robot_host] = robot_file_parser else: robot_file_map[robot_host] = None