# Veridian robots.txt file #=========================================================# # Block all web crawlers from accessing the site #=========================================================# User-agent: * Disallow: / # Applebot needs to be specifically told to stop; otherwise, it follows Googlebot User-agent: Applebot Disallow: / # Explicitly state that it is not for Google’s machine learning purposes User-agent: Google-Extended Disallow: / #=========================================================# # Only allow the following bots #=========================================================# User-agent: archive.org_bot Sitemap: https://cdnc.ucr.edu/sitemap-cdnc-index.xml Sitemap: https://cdnc.ucr.edu/sitemap-jewishweekly-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /?*&e=* # Block all internal links Disallow: /cgi-bin/jewishweekly?a=a # Block authentication Disallow: /cgi-bin/jewishweekly?a=cb # Block content blocking Disallow: /cgi-bin/jewishweekly?a=gs # Block graphserver requests Disallow: /cgi-bin/jewishweekly?a=is # Block imageserver requests Disallow: /cgi-bin/jewishweekly?a=pme # Block publication metadata editor Disallow: /cgi-bin/jewishweekly?a=q # Block queries Disallow: /cgi-bin/jewishweekly?a=tc # Block text correction Disallow: /cgi-bin/jewishweekly?a=ua # Block user account Disallow: /cgi-bin/jewishweekly?a=ur # Block user registration Disallow: /cgi-bin/jewishweekly?*&e=* # Block all internal links Disallow: */iiif/ # Block all IIIF requests User-agent: bingbot Sitemap: https://cdnc.ucr.edu/sitemap-cdnc-index.xml Sitemap: https://cdnc.ucr.edu/sitemap-jewishweekly-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /?*&e=* # Block all internal links Disallow: /cgi-bin/jewishweekly?a=a # Block authentication Disallow: /cgi-bin/jewishweekly?a=cb # Block content blocking Disallow: /cgi-bin/jewishweekly?a=gs # Block graphserver requests Disallow: /cgi-bin/jewishweekly?a=is # Block imageserver requests Disallow: /cgi-bin/jewishweekly?a=pme # Block publication metadata editor Disallow: /cgi-bin/jewishweekly?a=q # Block queries Disallow: /cgi-bin/jewishweekly?a=tc # Block text correction Disallow: /cgi-bin/jewishweekly?a=ua # Block user account Disallow: /cgi-bin/jewishweekly?a=ur # Block user registration Disallow: /cgi-bin/jewishweekly?*&e=* # Block all internal links Disallow: */iiif/ # Block all IIIF requests User-agent: Googlebot Sitemap: https://cdnc.ucr.edu/sitemap-cdnc-index.xml Sitemap: https://cdnc.ucr.edu/sitemap-jewishweekly-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /?*&e=* # Block all internal links Disallow: /cgi-bin/jewishweekly?a=a # Block authentication Disallow: /cgi-bin/jewishweekly?a=cb # Block content blocking Disallow: /cgi-bin/jewishweekly?a=gs # Block graphserver requests Disallow: /cgi-bin/jewishweekly?a=is # Block imageserver requests Disallow: /cgi-bin/jewishweekly?a=pme # Block publication metadata editor Disallow: /cgi-bin/jewishweekly?a=q # Block queries Disallow: /cgi-bin/jewishweekly?a=tc # Block text correction Disallow: /cgi-bin/jewishweekly?a=ua # Block user account Disallow: /cgi-bin/jewishweekly?a=ur # Block user registration Disallow: /cgi-bin/jewishweekly?*&e=* # Block all internal links Disallow: */iiif/ # Block all IIIF requests User-agent: Yahoo Sitemap: https://cdnc.ucr.edu/sitemap-cdnc-index.xml Sitemap: https://cdnc.ucr.edu/sitemap-jewishweekly-index.xml Crawl-delay: 1 # Wait one second between requests Disallow: /cgi-bin/imageserver.pl # Block imageserver requests Disallow: /?a=a # Block authentication Disallow: /?a=cb # Block content blocking Disallow: /?a=gs # Block graphserver requests Disallow: /?a=is # Block imageserver requests Disallow: /?a=pme # Block publication metadata editor Disallow: /?a=q # Block queries Disallow: /?a=tc # Block text correction Disallow: /?a=ua # Block user account Disallow: /?a=ur # Block user registration Disallow: /?*&e=* # Block all internal links Disallow: /cgi-bin/jewishweekly?a=a # Block authentication Disallow: /cgi-bin/jewishweekly?a=cb # Block content blocking Disallow: /cgi-bin/jewishweekly?a=gs # Block graphserver requests Disallow: /cgi-bin/jewishweekly?a=is # Block imageserver requests Disallow: /cgi-bin/jewishweekly?a=pme # Block publication metadata editor Disallow: /cgi-bin/jewishweekly?a=q # Block queries Disallow: /cgi-bin/jewishweekly?a=tc # Block text correction Disallow: /cgi-bin/jewishweekly?a=ua # Block user account Disallow: /cgi-bin/jewishweekly?a=ur # Block user registration Disallow: /cgi-bin/jewishweekly?*&e=* # Block all internal links Disallow: */iiif/ # Block all IIIF requests