# Veridian robots.txt file

User-agent: *
Sitemap: https://cdnc.ucr.edu/sitemap-cdnc-index.xml
Crawl-delay: 1                                  # Wait one second between requests
Disallow: /cgi-bin/imageserver.pl               # Block imageserver requests
Disallow: /cgi-bin/cdnc?a=a                     # Block authentication
Disallow: /cgi-bin/cdnc?a=is                    # Block imageserver requests
Disallow: /cgi-bin/cdnc?a=q                     # Block queries
Disallow: /cgi-bin/cdnc?a=tc                    # Block text correction
Disallow: /cgi-bin/cdnc?a=ur                    # Block user registration
Disallow: /cgi-bin/cdnc?*&e=*                   # Block all internal links
Disallow: /cgi-bin/jewishweekly?a=a             # Block authentication
Disallow: /cgi-bin/jewishweekly?a=is            # Block imageserver requests
Disallow: /cgi-bin/jewishweekly?a=q             # Block queries
Disallow: /cgi-bin/jewishweekly?a=tc            # Block text correction
Disallow: /cgi-bin/jewishweekly?a=ur            # Block user registration
Disallow: /cgi-bin/jewishweekly?*&e=*           # Block all internal links